From 49f3dadb8f6fcbf07824c8998932b963413babca Mon Sep 17 00:00:00 2001 From: Dan Stough Date: Thu, 14 Jul 2022 14:45:51 -0400 Subject: [PATCH] feat: connect proxy xDS for destinations Signed-off-by: Dhia Ayachi --- agent/agent.go | 4 + agent/cache-types/service_gateways.go | 52 ++ agent/cache-types/service_gateways_test.go | 57 +++ agent/consul/internal_endpoint.go | 50 ++ agent/consul/internal_endpoint_test.go | 476 ++++++++++++++++++ agent/consul/state/catalog.go | 21 +- agent/consul/state/catalog_test.go | 395 +++++++++++++++ agent/proxycfg-glue/glue.go | 12 + agent/proxycfg/connect_proxy.go | 88 ++++ agent/proxycfg/data_sources.go | 21 +- agent/proxycfg/internal/watch/watchmap.go | 15 + .../proxycfg/internal/watch/watchmap_test.go | 41 ++ agent/proxycfg/manager_test.go | 4 + agent/proxycfg/snapshot.go | 5 + agent/proxycfg/state.go | 3 + agent/proxycfg/state_test.go | 217 +++++++- agent/proxycfg/testing.go | 43 +- agent/proxycfg/testing_terminating_gateway.go | 60 ++- agent/proxycfg/testing_tproxy.go | 115 +++++ agent/xds/clusters.go | 195 ++++--- agent/xds/clusters_test.go | 6 - agent/xds/endpoints.go | 19 +- agent/xds/listeners.go | 204 ++++---- agent/xds/listeners_test.go | 6 - agent/xds/rbac.go | 14 +- agent/xds/resources_test.go | 16 + agent/xds/routes.go | 105 ++-- ...ransparent-proxy-destination.latest.golden | 255 ++++++++++ ...ng-gateway-destinations-only.latest.golden | 105 +++- ...ransparent-proxy-destination.latest.golden | 119 +++++ ...ng-gateway-destinations-only.latest.golden | 5 + ...ransparent-proxy-destination.latest.golden | 185 +++++++ ...roxy-dial-instances-directly.latest.golden | 24 +- ...nsparent-proxy-http-upstream.latest.golden | 24 +- ...ng-gateway-destinations-only.latest.golden | 202 +++++++- .../listeners/transparent-proxy.latest.golden | 24 +- ...ransparent-proxy-destination.latest.golden | 5 + ...ng-gateway-destinations-only.latest.golden | 53 ++ 38 files changed, 2897 insertions(+), 348 deletions(-) create mode 100644 agent/cache-types/service_gateways.go create mode 100644 agent/cache-types/service_gateways_test.go create mode 100644 agent/xds/testdata/clusters/transparent-proxy-destination.latest.golden create mode 100644 agent/xds/testdata/endpoints/transparent-proxy-destination.latest.golden create mode 100644 agent/xds/testdata/endpoints/transparent-proxy-terminating-gateway-destinations-only.latest.golden create mode 100644 agent/xds/testdata/listeners/transparent-proxy-destination.latest.golden create mode 100644 agent/xds/testdata/routes/transparent-proxy-destination.latest.golden create mode 100644 agent/xds/testdata/routes/transparent-proxy-terminating-gateway-destinations-only.latest.golden diff --git a/agent/agent.go b/agent/agent.go index 44157a91fc..5412436e56 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -4075,6 +4075,7 @@ func (a *Agent) registerCache() { a.cache.RegisterType(cachetype.IntentionMatchName, &cachetype.IntentionMatch{RPC: a}) a.cache.RegisterType(cachetype.IntentionUpstreamsName, &cachetype.IntentionUpstreams{RPC: a}) + a.cache.RegisterType(cachetype.IntentionUpstreamsDestinationName, &cachetype.IntentionUpstreamsDestination{RPC: a}) a.cache.RegisterType(cachetype.CatalogServicesName, &cachetype.CatalogServices{RPC: a}) @@ -4097,6 +4098,7 @@ func (a *Agent) registerCache() { a.cache.RegisterType(cachetype.CompiledDiscoveryChainName, &cachetype.CompiledDiscoveryChain{RPC: a}) a.cache.RegisterType(cachetype.GatewayServicesName, &cachetype.GatewayServices{RPC: a}) + a.cache.RegisterType(cachetype.ServiceGatewaysName, &cachetype.ServiceGateways{RPC: a}) a.cache.RegisterType(cachetype.ConfigEntryListName, &cachetype.ConfigEntryList{RPC: a}) @@ -4220,10 +4222,12 @@ func (a *Agent) proxyDataSources() proxycfg.DataSources { Datacenters: proxycfgglue.CacheDatacenters(a.cache), FederationStateListMeshGateways: proxycfgglue.CacheFederationStateListMeshGateways(a.cache), GatewayServices: proxycfgglue.CacheGatewayServices(a.cache), + ServiceGateways: proxycfgglue.CacheServiceGateways(a.cache), Health: proxycfgglue.ClientHealth(a.rpcClientHealth), HTTPChecks: proxycfgglue.CacheHTTPChecks(a.cache), Intentions: proxycfgglue.CacheIntentions(a.cache), IntentionUpstreams: proxycfgglue.CacheIntentionUpstreams(a.cache), + IntentionUpstreamsDestination: proxycfgglue.CacheIntentionUpstreamsDestination(a.cache), InternalServiceDump: proxycfgglue.CacheInternalServiceDump(a.cache), LeafCertificate: proxycfgglue.CacheLeafCertificate(a.cache), PeeredUpstreams: proxycfgglue.CachePeeredUpstreams(a.cache), diff --git a/agent/cache-types/service_gateways.go b/agent/cache-types/service_gateways.go new file mode 100644 index 0000000000..1c7a8e8557 --- /dev/null +++ b/agent/cache-types/service_gateways.go @@ -0,0 +1,52 @@ +package cachetype + +import ( + "fmt" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/agent/structs" +) + +// Recommended name for registration. +const ServiceGatewaysName = "service-gateways" + +// GatewayUpstreams supports fetching upstreams for a given gateway name. +type ServiceGateways struct { + RegisterOptionsBlockingRefresh + RPC RPC +} + +func (g *ServiceGateways) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { + var result cache.FetchResult + + // The request should be a ServiceSpecificRequest. + reqReal, ok := req.(*structs.ServiceSpecificRequest) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: request wrong type: %T", req) + } + + // Lightweight copy this object so that manipulating QueryOptions doesn't race. + dup := *reqReal + reqReal = &dup + + // Set the minimum query index to our current index so we block + reqReal.QueryOptions.MinQueryIndex = opts.MinIndex + reqReal.QueryOptions.MaxQueryTime = opts.Timeout + + // Always allow stale - there's no point in hitting leader if the request is + // going to be served from cache and end up arbitrarily stale anyway. This + // allows cached service-discover to automatically read scale across all + // servers too. + reqReal.AllowStale = true + + // Fetch + var reply structs.IndexedCheckServiceNodes + if err := g.RPC.RPC("Internal.ServiceGateways", reqReal, &reply); err != nil { + return result, err + } + + result.Value = &reply + result.Index = reply.QueryMeta.Index + return result, nil +} diff --git a/agent/cache-types/service_gateways_test.go b/agent/cache-types/service_gateways_test.go new file mode 100644 index 0000000000..39c6b474d2 --- /dev/null +++ b/agent/cache-types/service_gateways_test.go @@ -0,0 +1,57 @@ +package cachetype + +import ( + "testing" + "time" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/agent/structs" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestServiceGateways(t *testing.T) { + rpc := TestRPC(t) + typ := &ServiceGateways{RPC: rpc} + + // Expect the proper RPC call. This also sets the expected value + // since that is return-by-pointer in the arguments. + var resp *structs.IndexedCheckServiceNodes + rpc.On("RPC", "Internal.ServiceGateways", mock.Anything, mock.Anything).Return(nil). + Run(func(args mock.Arguments) { + req := args.Get(1).(*structs.ServiceSpecificRequest) + require.Equal(t, uint64(24), req.QueryOptions.MinQueryIndex) + require.Equal(t, 1*time.Second, req.QueryOptions.MaxQueryTime) + require.True(t, req.AllowStale) + require.Equal(t, "foo", req.ServiceName) + + nodes := []structs.CheckServiceNode{ + { + Service: &structs.NodeService{ + Tags: req.ServiceTags, + }, + }, + } + + reply := args.Get(2).(*structs.IndexedCheckServiceNodes) + reply.Nodes = nodes + reply.QueryMeta.Index = 48 + resp = reply + }) + + // Fetch + resultA, err := typ.Fetch(cache.FetchOptions{ + MinIndex: 24, + Timeout: 1 * time.Second, + }, &structs.ServiceSpecificRequest{ + Datacenter: "dc1", + ServiceName: "foo", + }) + require.NoError(t, err) + require.Equal(t, cache.FetchResult{ + Value: resp, + Index: 48, + }, resultA) + + rpc.AssertExpectations(t) +} diff --git a/agent/consul/internal_endpoint.go b/agent/consul/internal_endpoint.go index a041c7eeb6..8f44c0f7a9 100644 --- a/agent/consul/internal_endpoint.go +++ b/agent/consul/internal_endpoint.go @@ -453,6 +453,56 @@ func (m *Internal) GatewayServiceDump(args *structs.ServiceSpecificRequest, repl return err } +// ServiceGateways returns all the nodes for services associated with a gateway along with their gateway config +func (m *Internal) ServiceGateways(args *structs.ServiceSpecificRequest, reply *structs.IndexedCheckServiceNodes) error { + if done, err := m.srv.ForwardRPC("Internal.ServiceGateways", args, reply); done { + return err + } + + // Verify the arguments + if args.ServiceName == "" { + return fmt.Errorf("Must provide gateway name") + } + + var authzContext acl.AuthorizerContext + authz, err := m.srv.ResolveTokenAndDefaultMeta(args.Token, &args.EnterpriseMeta, &authzContext) + if err != nil { + return err + } + + if err := m.srv.validateEnterpriseRequest(&args.EnterpriseMeta, false); err != nil { + return err + } + + // We need read access to the service we're trying to find gateways for, so check that first. + if err := authz.ToAllowAuthorizer().ServiceReadAllowed(args.ServiceName, &authzContext); err != nil { + return err + } + + err = m.srv.blockingQuery( + &args.QueryOptions, + &reply.QueryMeta, + func(ws memdb.WatchSet, state *state.Store) error { + var maxIdx uint64 + idx, gateways, err := state.ServiceGateways(ws, args.ServiceName, args.ServiceKind, args.EnterpriseMeta) + if err != nil { + return err + } + if idx > maxIdx { + maxIdx = idx + } + + reply.Index, reply.Nodes = maxIdx, gateways + + if err := m.srv.filterACL(args.Token, reply); err != nil { + return err + } + return nil + }) + + return err +} + // GatewayIntentions Match returns the set of intentions that match the given source/destination. func (m *Internal) GatewayIntentions(args *structs.IntentionQueryRequest, reply *structs.IndexedIntentions) error { // Forward if necessary diff --git a/agent/consul/internal_endpoint_test.go b/agent/consul/internal_endpoint_test.go index 7d7d421c83..f02150b8ca 100644 --- a/agent/consul/internal_endpoint_test.go +++ b/agent/consul/internal_endpoint_test.go @@ -2811,3 +2811,479 @@ func TestInternal_PeeredUpstreams(t *testing.T) { } require.Equal(t, expect, out.Services) } + +func TestInternal_ServiceGatewayService_Terminating(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + codec := rpcClient(t, s1) + defer codec.Close() + + testrpc.WaitForTestAgent(t, s1.RPC, "dc1") + + db := structs.NodeService{ + ID: "db2", + Service: "db", + } + + redis := structs.NodeService{ + ID: "redis", + Service: "redis", + } + + // Register gateway and two service instances that will be associated with it + { + arg := structs.RegisterRequest{ + Datacenter: "dc1", + Node: "foo", + Address: "10.1.2.2", + Service: &structs.NodeService{ + ID: "terminating-gateway-01", + Service: "terminating-gateway", + Kind: structs.ServiceKindTerminatingGateway, + Port: 443, + Address: "198.18.1.3", + }, + Check: &structs.HealthCheck{ + Name: "terminating connect", + Status: api.HealthPassing, + ServiceID: "terminating-gateway-01", + }, + } + var out struct{} + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + + arg = structs.RegisterRequest{ + Datacenter: "dc1", + Node: "bar", + Address: "127.0.0.2", + Service: &structs.NodeService{ + ID: "db", + Service: "db", + }, + Check: &structs.HealthCheck{ + Name: "db-warning", + Status: api.HealthWarning, + ServiceID: "db", + }, + } + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + + arg = structs.RegisterRequest{ + Datacenter: "dc1", + Node: "baz", + Address: "127.0.0.3", + Service: &db, + Check: &structs.HealthCheck{ + Name: "db2-passing", + Status: api.HealthPassing, + ServiceID: "db2", + }, + } + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + } + + // Register terminating-gateway config entry, linking it to db and redis (dne) + { + args := &structs.TerminatingGatewayConfigEntry{ + Name: "terminating-gateway", + Kind: structs.TerminatingGateway, + Services: []structs.LinkedService{ + { + Name: "db", + }, + { + Name: "redis", + CAFile: "/etc/certs/ca.pem", + CertFile: "/etc/certs/cert.pem", + KeyFile: "/etc/certs/key.pem", + }, + }, + } + + req := structs.ConfigEntryRequest{ + Op: structs.ConfigEntryUpsert, + Datacenter: "dc1", + Entry: args, + } + var configOutput bool + require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &req, &configOutput)) + require.True(t, configOutput) + } + + var out structs.IndexedCheckServiceNodes + req := structs.ServiceSpecificRequest{ + Datacenter: "dc1", + ServiceName: "db", + ServiceKind: structs.ServiceKindTerminatingGateway, + } + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Internal.ServiceGateways", &req, &out)) + + for _, n := range out.Nodes { + n.Node.RaftIndex = structs.RaftIndex{} + n.Service.RaftIndex = structs.RaftIndex{} + for _, m := range n.Checks { + m.RaftIndex = structs.RaftIndex{} + } + } + + expect := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + Node: "foo", + RaftIndex: structs.RaftIndex{}, + Address: "10.1.2.2", + Datacenter: "dc1", + Partition: acl.DefaultPartitionName, + }, + Service: &structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + ID: "terminating-gateway-01", + Service: "terminating-gateway", + TaggedAddresses: map[string]structs.ServiceAddress{ + "consul-virtual:" + db.CompoundServiceName().String(): {Address: "240.0.0.1"}, + "consul-virtual:" + redis.CompoundServiceName().String(): {Address: "240.0.0.2"}, + }, + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + Tags: []string{}, + Meta: map[string]string{}, + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + RaftIndex: structs.RaftIndex{}, + Address: "198.18.1.3", + }, + Checks: structs.HealthChecks{ + &structs.HealthCheck{ + Name: "terminating connect", + Node: "foo", + CheckID: "terminating connect", + Status: api.HealthPassing, + ServiceID: "terminating-gateway-01", + ServiceName: "terminating-gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + }, + }, + }, + } + + assert.Equal(t, expect, out.Nodes) +} + +func TestInternal_ServiceGatewayService_Terminating_ACL(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + dir1, s1 := testServerWithConfig(t, func(c *Config) { + c.PrimaryDatacenter = "dc1" + c.ACLsEnabled = true + c.ACLInitialManagementToken = "root" + c.ACLResolverSettings.ACLDefaultPolicy = "deny" + }) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + codec := rpcClient(t, s1) + defer codec.Close() + + testrpc.WaitForTestAgent(t, s1.RPC, "dc1", testrpc.WithToken("root")) + + // Create the ACL. + token, err := upsertTestTokenWithPolicyRules(codec, "root", "dc1", ` + service "db" { policy = "read" } + service "terminating-gateway" { policy = "read" } + node_prefix "" { policy = "read" }`) + require.NoError(t, err) + + // Register gateway and two service instances that will be associated with it + { + arg := structs.RegisterRequest{ + Datacenter: "dc1", + Node: "foo", + Address: "127.0.0.1", + Service: &structs.NodeService{ + ID: "terminating-gateway", + Service: "terminating-gateway", + Kind: structs.ServiceKindTerminatingGateway, + Port: 443, + }, + Check: &structs.HealthCheck{ + Name: "terminating connect", + Status: api.HealthPassing, + ServiceID: "terminating-gateway", + }, + WriteRequest: structs.WriteRequest{Token: "root"}, + } + var out struct{} + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + { + arg := structs.RegisterRequest{ + Datacenter: "dc1", + Node: "foo", + Address: "127.0.0.1", + Service: &structs.NodeService{ + ID: "terminating-gateway2", + Service: "terminating-gateway2", + Kind: structs.ServiceKindTerminatingGateway, + Port: 444, + }, + Check: &structs.HealthCheck{ + Name: "terminating connect", + Status: api.HealthPassing, + ServiceID: "terminating-gateway2", + }, + WriteRequest: structs.WriteRequest{Token: "root"}, + } + var out struct{} + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + } + + arg = structs.RegisterRequest{ + Datacenter: "dc1", + Node: "bar", + Address: "127.0.0.2", + Service: &structs.NodeService{ + ID: "db", + Service: "db", + }, + Check: &structs.HealthCheck{ + Name: "db-warning", + Status: api.HealthWarning, + ServiceID: "db", + }, + WriteRequest: structs.WriteRequest{Token: "root"}, + } + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + + arg = structs.RegisterRequest{ + Datacenter: "dc1", + Node: "baz", + Address: "127.0.0.3", + Service: &structs.NodeService{ + ID: "api", + Service: "api", + }, + Check: &structs.HealthCheck{ + Name: "api-passing", + Status: api.HealthPassing, + ServiceID: "api", + }, + WriteRequest: structs.WriteRequest{Token: "root"}, + } + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + } + + // Register terminating-gateway config entry, linking it to db and api + { + args := &structs.TerminatingGatewayConfigEntry{ + Name: "terminating-gateway", + Kind: structs.TerminatingGateway, + Services: []structs.LinkedService{ + {Name: "db"}, + {Name: "api"}, + }, + } + + req := structs.ConfigEntryRequest{ + Op: structs.ConfigEntryUpsert, + Datacenter: "dc1", + Entry: args, + WriteRequest: structs.WriteRequest{Token: "root"}, + } + var out bool + require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &req, &out)) + require.True(t, out) + } + + // Register terminating-gateway config entry, linking it to db and api + { + args := &structs.TerminatingGatewayConfigEntry{ + Name: "terminating-gateway2", + Kind: structs.TerminatingGateway, + Services: []structs.LinkedService{ + {Name: "db"}, + {Name: "api"}, + }, + } + + req := structs.ConfigEntryRequest{ + Op: structs.ConfigEntryUpsert, + Datacenter: "dc1", + Entry: args, + WriteRequest: structs.WriteRequest{Token: "root"}, + } + var out bool + require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &req, &out)) + require.True(t, out) + } + + var out structs.IndexedCheckServiceNodes + + // Not passing a token with service:read on Gateway leads to PermissionDenied + req := structs.ServiceSpecificRequest{ + Datacenter: "dc1", + ServiceName: "db", + ServiceKind: structs.ServiceKindTerminatingGateway, + } + err = msgpackrpc.CallWithCodec(codec, "Internal.ServiceGateways", &req, &out) + require.Error(t, err, acl.ErrPermissionDenied) + + // Passing a token without service:read on api leads to it getting filtered out + req = structs.ServiceSpecificRequest{ + Datacenter: "dc1", + ServiceName: "db", + ServiceKind: structs.ServiceKindTerminatingGateway, + QueryOptions: structs.QueryOptions{Token: token.SecretID}, + } + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Internal.ServiceGateways", &req, &out)) + + nodes := out.Nodes + require.Len(t, nodes, 1) + require.Equal(t, "foo", nodes[0].Node.Node) + require.Equal(t, structs.ServiceKindTerminatingGateway, nodes[0].Service.Kind) + require.Equal(t, "terminating-gateway", nodes[0].Service.Service) + require.Equal(t, "terminating-gateway", nodes[0].Service.ID) + require.True(t, out.QueryMeta.ResultsFilteredByACLs, "ResultsFilteredByACLs should be true") +} + +func TestInternal_ServiceGatewayService_Terminating_Destination(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + codec := rpcClient(t, s1) + defer codec.Close() + + testrpc.WaitForTestAgent(t, s1.RPC, "dc1") + + google := structs.NodeService{ + ID: "google", + Service: "google", + } + + // Register service-default with conflicting destination address + { + arg := structs.ConfigEntryRequest{ + Op: structs.ConfigEntryUpsert, + Datacenter: "dc1", + Entry: &structs.ServiceConfigEntry{ + Name: "google", + Destination: &structs.DestinationConfig{Address: "www.google.com", Port: 443}, + EnterpriseMeta: *acl.DefaultEnterpriseMeta(), + }, + } + var configOutput bool + require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &arg, &configOutput)) + require.True(t, configOutput) + } + + // Register terminating-gateway config entry, linking it to google.com + { + arg := structs.RegisterRequest{ + Datacenter: "dc1", + Node: "foo", + Address: "127.0.0.1", + Service: &structs.NodeService{ + ID: "terminating-gateway", + Service: "terminating-gateway", + Kind: structs.ServiceKindTerminatingGateway, + Port: 443, + }, + Check: &structs.HealthCheck{ + Name: "terminating connect", + Status: api.HealthPassing, + ServiceID: "terminating-gateway", + }, + } + var out struct{} + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out)) + } + { + args := &structs.TerminatingGatewayConfigEntry{ + Name: "terminating-gateway", + Kind: structs.TerminatingGateway, + Services: []structs.LinkedService{ + { + Name: "google", + }, + }, + } + + req := structs.ConfigEntryRequest{ + Op: structs.ConfigEntryUpsert, + Datacenter: "dc1", + Entry: args, + } + var configOutput bool + require.NoError(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &req, &configOutput)) + require.True(t, configOutput) + } + + var out structs.IndexedCheckServiceNodes + req := structs.ServiceSpecificRequest{ + Datacenter: "dc1", + ServiceName: "google", + ServiceKind: structs.ServiceKindTerminatingGateway, + } + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Internal.ServiceGateways", &req, &out)) + + nodes := out.Nodes + + for _, n := range nodes { + n.Node.RaftIndex = structs.RaftIndex{} + n.Service.RaftIndex = structs.RaftIndex{} + for _, m := range n.Checks { + m.RaftIndex = structs.RaftIndex{} + } + } + + expect := structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + Node: "foo", + RaftIndex: structs.RaftIndex{}, + Address: "127.0.0.1", + Datacenter: "dc1", + Partition: acl.DefaultPartitionName, + }, + Service: &structs.NodeService{ + Kind: structs.ServiceKindTerminatingGateway, + ID: "terminating-gateway", + Service: "terminating-gateway", + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + Tags: []string{}, + Meta: map[string]string{}, + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + TaggedAddresses: map[string]structs.ServiceAddress{ + "consul-virtual:" + google.CompoundServiceName().String(): {Address: "240.0.0.1"}, + }, + RaftIndex: structs.RaftIndex{}, + Address: "", + }, + Checks: structs.HealthChecks{ + &structs.HealthCheck{ + Name: "terminating connect", + Node: "foo", + CheckID: "terminating connect", + Status: api.HealthPassing, + ServiceID: "terminating-gateway", + ServiceName: "terminating-gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + }, + }, + }, + } + + assert.Len(t, nodes, 1) + assert.Equal(t, expect, nodes) +} diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index 2f116ab0f9..622cccd358 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -2907,6 +2907,25 @@ func (s *Store) GatewayServices(ws memdb.WatchSet, gateway string, entMeta *acl. return lib.MaxUint64(maxIdx, idx), results, nil } +// TODO: Find a way to consolidate this with CheckIngressServiceNodes +// ServiceGateways is used to query all gateways associated with a service +func (s *Store) ServiceGateways(ws memdb.WatchSet, service string, kind structs.ServiceKind, entMeta acl.EnterpriseMeta) (uint64, structs.CheckServiceNodes, error) { + tx := s.db.Txn(false) + defer tx.Abort() + + // tableGatewayServices is not peer-aware, and the existence of TG/IG gateways is scrubbed during peer replication. + maxIdx, nodes, err := serviceGatewayNodes(tx, ws, service, kind, &entMeta, structs.DefaultPeerKeyword) + + // Watch for index changes to the gateway nodes + idx, chans := maxIndexAndWatchChsForServiceNodes(tx, nodes, false) + for _, ch := range chans { + ws.Add(ch) + } + maxIdx = lib.MaxUint64(maxIdx, idx) + + return parseCheckServiceNodes(tx, ws, maxIdx, nodes, &entMeta, structs.DefaultPeerKeyword, err) +} + func (s *Store) VirtualIPForService(psn structs.PeeredServiceName) (string, error) { tx := s.db.Txn(false) defer tx.Abort() @@ -3862,7 +3881,7 @@ func (s *Store) collectGatewayServices(tx ReadTxn, ws memdb.WatchSet, iter memdb return maxIdx, results, nil } -// TODO(ingress): How to handle index rolling back when a config entry is +// TODO: How to handle index rolling back when a config entry is // deleted that references a service? // We might need something like the service_last_extinction index? func serviceGatewayNodes(tx ReadTxn, ws memdb.WatchSet, service string, kind structs.ServiceKind, entMeta *acl.EnterpriseMeta, peerName string) (uint64, structs.ServiceNodes, error) { diff --git a/agent/consul/state/catalog_test.go b/agent/consul/state/catalog_test.go index d2a970b075..fed3bd0ee3 100644 --- a/agent/consul/state/catalog_test.go +++ b/agent/consul/state/catalog_test.go @@ -4,6 +4,7 @@ import ( "context" crand "crypto/rand" "fmt" + "github.com/hashicorp/consul/acl" "reflect" "sort" "strings" @@ -5346,6 +5347,400 @@ func TestStateStore_GatewayServices_Terminating(t *testing.T) { assert.Len(t, out, 0) } +func TestStateStore_ServiceGateways_Terminating(t *testing.T) { + s := testStateStore(t) + + // Listing with no results returns an empty list. + ws := memdb.NewWatchSet() + idx, nodes, err := s.GatewayServices(ws, "db", nil) + assert.Nil(t, err) + assert.Equal(t, uint64(0), idx) + assert.Len(t, nodes, 0) + + // Create some nodes + assert.Nil(t, s.EnsureNode(10, &structs.Node{Node: "foo", Address: "127.0.0.1"})) + assert.Nil(t, s.EnsureNode(11, &structs.Node{Node: "bar", Address: "127.0.0.2"})) + assert.Nil(t, s.EnsureNode(12, &structs.Node{Node: "baz", Address: "127.0.0.2"})) + + // Typical services and some consul services spread across two nodes + assert.Nil(t, s.EnsureService(13, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: nil, Address: "", Port: 5000})) + assert.Nil(t, s.EnsureService(15, "bar", &structs.NodeService{ID: "api", Service: "api", Tags: nil, Address: "", Port: 5000})) + assert.Nil(t, s.EnsureService(16, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil})) + assert.Nil(t, s.EnsureService(17, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil})) + + // Add ingress gateway and a connect proxy, neither should get picked up by terminating gateway + ingressNS := &structs.NodeService{ + Kind: structs.ServiceKindIngressGateway, + ID: "ingress", + Service: "ingress", + Port: 8443, + } + assert.Nil(t, s.EnsureService(18, "baz", ingressNS)) + + proxyNS := &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "db proxy", + Service: "db proxy", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "db", + }, + Port: 8000, + } + assert.Nil(t, s.EnsureService(19, "foo", proxyNS)) + + // Register a gateway + assert.Nil(t, s.EnsureService(20, "baz", &structs.NodeService{Kind: structs.ServiceKindTerminatingGateway, ID: "gateway", Service: "gateway", Port: 443})) + + // Associate gateway with db and api + assert.Nil(t, s.EnsureConfigEntry(21, &structs.TerminatingGatewayConfigEntry{ + Kind: "terminating-gateway", + Name: "gateway", + Services: []structs.LinkedService{ + { + Name: "db", + }, + { + Name: "api", + }, + }, + })) + assert.True(t, watchFired(ws)) + + // Read everything back. + ws = memdb.NewWatchSet() + idx, out, err := s.ServiceGateways(ws, "db", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + assert.Equal(t, uint64(21), idx) + assert.Len(t, out, 1) + + expect := structs.CheckServiceNodes{ + { + Node: &structs.Node{ + ID: "", + Address: "127.0.0.2", + Node: "baz", + Partition: acl.DefaultPartitionName, + RaftIndex: structs.RaftIndex{ + CreateIndex: 12, + ModifyIndex: 12, + }, + }, + Service: &structs.NodeService{ + Service: "gateway", + Kind: structs.ServiceKindTerminatingGateway, + ID: "gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + RaftIndex: structs.RaftIndex{ + CreateIndex: 20, + ModifyIndex: 20, + }, + }, + }, + } + assert.Equal(t, expect, out) + + // Check that we don't update on same exact config + assert.Nil(t, s.EnsureConfigEntry(21, &structs.TerminatingGatewayConfigEntry{ + Kind: "terminating-gateway", + Name: "gateway", + Services: []structs.LinkedService{ + { + Name: "db", + }, + { + Name: "api", + }, + }, + })) + assert.False(t, watchFired(ws)) + + idx, out, err = s.ServiceGateways(ws, "api", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + assert.Equal(t, uint64(21), idx) + assert.Len(t, out, 1) + + expect = structs.CheckServiceNodes{ + { + Node: &structs.Node{ + ID: "", + Address: "127.0.0.2", + Node: "baz", + Partition: acl.DefaultPartitionName, + RaftIndex: structs.RaftIndex{ + CreateIndex: 12, + ModifyIndex: 12, + }, + }, + Service: &structs.NodeService{ + Service: "gateway", + Kind: structs.ServiceKindTerminatingGateway, + ID: "gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + RaftIndex: structs.RaftIndex{ + CreateIndex: 20, + ModifyIndex: 20, + }, + }, + }, + } + assert.Equal(t, expect, out) + + // Associate gateway with a wildcard and add TLS config + assert.Nil(t, s.EnsureConfigEntry(22, &structs.TerminatingGatewayConfigEntry{ + Kind: "terminating-gateway", + Name: "gateway", + Services: []structs.LinkedService{ + { + Name: "api", + CAFile: "api/ca.crt", + CertFile: "api/client.crt", + KeyFile: "api/client.key", + SNI: "my-domain", + }, + { + Name: "db", + }, + { + Name: "*", + CAFile: "ca.crt", + CertFile: "client.crt", + KeyFile: "client.key", + SNI: "my-alt-domain", + }, + }, + })) + assert.True(t, watchFired(ws)) + + // Read everything back. + ws = memdb.NewWatchSet() + idx, out, err = s.ServiceGateways(ws, "db", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + assert.Equal(t, uint64(22), idx) + assert.Len(t, out, 1) + + expect = structs.CheckServiceNodes{ + { + Node: &structs.Node{ + ID: "", + Address: "127.0.0.2", + Node: "baz", + Partition: acl.DefaultPartitionName, + RaftIndex: structs.RaftIndex{ + CreateIndex: 12, + ModifyIndex: 12, + }, + }, + Service: &structs.NodeService{ + Service: "gateway", + Kind: structs.ServiceKindTerminatingGateway, + ID: "gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + RaftIndex: structs.RaftIndex{ + CreateIndex: 20, + ModifyIndex: 20, + }, + }, + }, + } + assert.Equal(t, expect, out) + + // Add a service covered by wildcard + assert.Nil(t, s.EnsureService(23, "bar", &structs.NodeService{ID: "redis", Service: "redis", Tags: nil, Address: "", Port: 6379})) + + ws = memdb.NewWatchSet() + idx, out, err = s.ServiceGateways(ws, "redis", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + assert.Equal(t, uint64(23), idx) + assert.Len(t, out, 1) + + expect = structs.CheckServiceNodes{ + { + Node: &structs.Node{ + ID: "", + Address: "127.0.0.2", + Node: "baz", + Partition: acl.DefaultPartitionName, + RaftIndex: structs.RaftIndex{ + CreateIndex: 12, + ModifyIndex: 12, + }, + }, + Service: &structs.NodeService{ + Service: "gateway", + Kind: structs.ServiceKindTerminatingGateway, + ID: "gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + RaftIndex: structs.RaftIndex{ + CreateIndex: 20, + ModifyIndex: 20, + }, + }, + }, + } + assert.Equal(t, expect, out) + + // Delete a service covered by wildcard + assert.Nil(t, s.DeleteService(24, "bar", "redis", structs.DefaultEnterpriseMetaInDefaultPartition(), "")) + assert.True(t, watchFired(ws)) + + ws = memdb.NewWatchSet() + idx, out, err = s.ServiceGateways(ws, "redis", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + // TODO: wildcards don't keep the same extinction index + assert.Equal(t, uint64(0), idx) + assert.Len(t, out, 0) + + // Update the entry that only leaves one service + assert.Nil(t, s.EnsureConfigEntry(25, &structs.TerminatingGatewayConfigEntry{ + Kind: "terminating-gateway", + Name: "gateway", + Services: []structs.LinkedService{ + { + Name: "db", + }, + }, + })) + assert.True(t, watchFired(ws)) + + ws = memdb.NewWatchSet() + idx, out, err = s.ServiceGateways(ws, "db", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + assert.Equal(t, uint64(25), idx) + assert.Len(t, out, 1) + + // previously associated services should not be present + expect = structs.CheckServiceNodes{ + { + Node: &structs.Node{ + ID: "", + Address: "127.0.0.2", + Node: "baz", + Partition: acl.DefaultPartitionName, + RaftIndex: structs.RaftIndex{ + CreateIndex: 12, + ModifyIndex: 12, + }, + }, + Service: &structs.NodeService{ + Service: "gateway", + Kind: structs.ServiceKindTerminatingGateway, + ID: "gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + RaftIndex: structs.RaftIndex{ + CreateIndex: 20, + ModifyIndex: 20, + }, + }, + }, + } + assert.Equal(t, expect, out) + + // Attempt to associate a different gateway with services that include db + assert.Nil(t, s.EnsureConfigEntry(26, &structs.TerminatingGatewayConfigEntry{ + Kind: "terminating-gateway", + Name: "gateway2", + Services: []structs.LinkedService{ + { + Name: "*", + }, + }, + })) + + // check that watchset fired for new terminating gateway node service + assert.Nil(t, s.EnsureService(20, "baz", &structs.NodeService{Kind: structs.ServiceKindTerminatingGateway, ID: "gateway2", Service: "gateway2", Port: 443})) + assert.True(t, watchFired(ws)) + + ws = memdb.NewWatchSet() + idx, out, err = s.ServiceGateways(ws, "db", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + assert.Equal(t, uint64(26), idx) + assert.Len(t, out, 2) + + expect = structs.CheckServiceNodes{ + { + Node: &structs.Node{ + ID: "", + Address: "127.0.0.2", + Node: "baz", + Partition: acl.DefaultPartitionName, + RaftIndex: structs.RaftIndex{ + CreateIndex: 12, + ModifyIndex: 12, + }, + }, + Service: &structs.NodeService{ + Service: "gateway", + Kind: structs.ServiceKindTerminatingGateway, + ID: "gateway", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + RaftIndex: structs.RaftIndex{ + CreateIndex: 20, + ModifyIndex: 20, + }, + }, + }, + { + Node: &structs.Node{ + ID: "", + Address: "127.0.0.2", + Node: "baz", + Partition: acl.DefaultPartitionName, + RaftIndex: structs.RaftIndex{ + CreateIndex: 12, + ModifyIndex: 12, + }, + }, + Service: &structs.NodeService{ + Service: "gateway2", + Kind: structs.ServiceKindTerminatingGateway, + ID: "gateway2", + EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), + Weights: &structs.Weights{Passing: 1, Warning: 1}, + Port: 443, + RaftIndex: structs.RaftIndex{ + CreateIndex: 20, + ModifyIndex: 20, + }, + }, + }, + } + assert.Equal(t, expect, out) + + // Deleting the all gateway's node services should trigger the watch and keep the raft index stable + assert.Nil(t, s.DeleteService(27, "baz", "gateway", structs.DefaultEnterpriseMetaInDefaultPartition(), structs.DefaultPeerKeyword)) + assert.True(t, watchFired(ws)) + assert.Nil(t, s.DeleteService(28, "baz", "gateway2", structs.DefaultEnterpriseMetaInDefaultPartition(), structs.DefaultPeerKeyword)) + + ws = memdb.NewWatchSet() + idx, out, err = s.ServiceGateways(ws, "db", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + assert.Equal(t, uint64(28), idx) + assert.Len(t, out, 0) + + // Deleting the config entry even with a node service should remove existing mappings + assert.Nil(t, s.EnsureService(29, "baz", &structs.NodeService{Kind: structs.ServiceKindTerminatingGateway, ID: "gateway", Service: "gateway", Port: 443})) + assert.Nil(t, s.DeleteConfigEntry(30, "terminating-gateway", "gateway", nil)) + assert.True(t, watchFired(ws)) + + idx, out, err = s.ServiceGateways(ws, "api", structs.ServiceKindTerminatingGateway, *structs.DefaultEnterpriseMetaInDefaultPartition()) + assert.Nil(t, err) + // TODO: similar to ingress, the index can backslide if the config is deleted. + assert.Equal(t, uint64(28), idx) + assert.Len(t, out, 0) +} + func TestStateStore_GatewayServices_ServiceDeletion(t *testing.T) { s := testStateStore(t) diff --git a/agent/proxycfg-glue/glue.go b/agent/proxycfg-glue/glue.go index 06798939bd..7c3311f363 100644 --- a/agent/proxycfg-glue/glue.go +++ b/agent/proxycfg-glue/glue.go @@ -54,6 +54,12 @@ func CacheDatacenters(c *cache.Cache) proxycfg.Datacenters { return &cacheProxyDataSource[*structs.DatacentersRequest]{c, cachetype.CatalogDatacentersName} } +// CacheServiceGateways satisfies the proxycfg.ServiceGateways interface by +// sourcing data from the agent cache. +func CacheServiceGateways(c *cache.Cache) proxycfg.GatewayServices { + return &cacheProxyDataSource[*structs.ServiceSpecificRequest]{c, cachetype.ServiceGatewaysName} +} + // CacheHTTPChecks satisifies the proxycfg.HTTPChecks interface by sourcing // data from the agent cache. func CacheHTTPChecks(c *cache.Cache) proxycfg.HTTPChecks { @@ -66,6 +72,12 @@ func CacheIntentionUpstreams(c *cache.Cache) proxycfg.IntentionUpstreams { return &cacheProxyDataSource[*structs.ServiceSpecificRequest]{c, cachetype.IntentionUpstreamsName} } +// CacheIntentionUpstreamsDestination satisfies the proxycfg.IntentionUpstreamsDestination interface +// by sourcing data from the agent cache. +func CacheIntentionUpstreamsDestination(c *cache.Cache) proxycfg.IntentionUpstreams { + return &cacheProxyDataSource[*structs.ServiceSpecificRequest]{c, cachetype.IntentionUpstreamsDestinationName} +} + // CacheInternalServiceDump satisfies the proxycfg.InternalServiceDump // interface by sourcing data from the agent cache. func CacheInternalServiceDump(c *cache.Cache) proxycfg.InternalServiceDump { diff --git a/agent/proxycfg/connect_proxy.go b/agent/proxycfg/connect_proxy.go index 3221150dba..823f7d9eff 100644 --- a/agent/proxycfg/connect_proxy.go +++ b/agent/proxycfg/connect_proxy.go @@ -28,10 +28,12 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e snap.ConnectProxy.WatchedGatewayEndpoints = make(map[UpstreamID]map[string]structs.CheckServiceNodes) snap.ConnectProxy.WatchedServiceChecks = make(map[structs.ServiceID][]structs.CheckType) snap.ConnectProxy.PreparedQueryEndpoints = make(map[UpstreamID]structs.CheckServiceNodes) + snap.ConnectProxy.DestinationsUpstream = watch.NewMap[UpstreamID, *structs.ServiceConfigEntry]() snap.ConnectProxy.UpstreamConfig = make(map[UpstreamID]*structs.Upstream) snap.ConnectProxy.PassthroughUpstreams = make(map[UpstreamID]map[string]map[string]struct{}) snap.ConnectProxy.PassthroughIndices = make(map[string]indexedTarget) snap.ConnectProxy.PeerUpstreamEndpoints = watch.NewMap[UpstreamID, structs.CheckServiceNodes]() + snap.ConnectProxy.DestinationGateways = watch.NewMap[UpstreamID, structs.CheckServiceNodes]() snap.ConnectProxy.PeerUpstreamEndpointsUseHostnames = make(map[UpstreamID]struct{}) // Watch for root changes @@ -116,6 +118,16 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e if err != nil { return snap, err } + // We also infer upstreams from destinations (egress points) + err = s.dataSources.IntentionUpstreamsDestination.Notify(ctx, &structs.ServiceSpecificRequest{ + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + ServiceName: s.proxyCfg.DestinationServiceName, + EnterpriseMeta: s.proxyID.EnterpriseMeta, + }, intentionUpstreamsDestinationID, s.ch) + if err != nil { + return snap, err + } } // Watch for updates to service endpoints for all upstreams @@ -508,7 +520,83 @@ func (s *handlerConnectProxy) handleUpdate(ctx context.Context, u UpdateEvent, s delete(snap.ConnectProxy.DiscoveryChain, uid) } } + case u.CorrelationID == intentionUpstreamsDestinationID: + resp, ok := u.Result.(*structs.IndexedServiceList) + if !ok { + return fmt.Errorf("invalid type for response %T", u.Result) + } + seenUpstreams := make(map[UpstreamID]struct{}) + for _, svc := range resp.Services { + uid := NewUpstreamIDFromServiceName(svc) + seenUpstreams[uid] = struct{}{} + { + childCtx, cancel := context.WithCancel(ctx) + err := s.dataSources.ConfigEntry.Notify(childCtx, &structs.ConfigEntryQuery{ + Kind: structs.ServiceDefaults, + Name: svc.Name, + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + EnterpriseMeta: svc.EnterpriseMeta, + }, DestinationConfigEntryID+svc.String(), s.ch) + if err != nil { + cancel() + return err + } + snap.ConnectProxy.DestinationsUpstream.InitWatch(uid, cancel) + } + { + childCtx, cancel := context.WithCancel(ctx) + err := s.dataSources.ServiceGateways.Notify(childCtx, &structs.ServiceSpecificRequest{ + ServiceName: svc.Name, + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + EnterpriseMeta: svc.EnterpriseMeta, + ServiceKind: structs.ServiceKindTerminatingGateway, + }, DestinationGatewayID+svc.String(), s.ch) + if err != nil { + cancel() + return err + } + snap.ConnectProxy.DestinationGateways.InitWatch(uid, cancel) + } + } + snap.ConnectProxy.DestinationsUpstream.ForEachKey(func(uid UpstreamID) bool { + if _, ok := seenUpstreams[uid]; !ok { + snap.ConnectProxy.DestinationsUpstream.CancelWatch(uid) + } + return true + }) + + snap.ConnectProxy.DestinationGateways.ForEachKey(func(uid UpstreamID) bool { + if _, ok := seenUpstreams[uid]; !ok { + snap.ConnectProxy.DestinationGateways.CancelWatch(uid) + } + return true + }) + case strings.HasPrefix(u.CorrelationID, DestinationConfigEntryID): + resp, ok := u.Result.(*structs.ConfigEntryResponse) + if !ok { + return fmt.Errorf("invalid type for response: %T", u.Result) + } + + pq := strings.TrimPrefix(u.CorrelationID, DestinationConfigEntryID) + uid := UpstreamIDFromString(pq) + serviceConf, ok := resp.Entry.(*structs.ServiceConfigEntry) + if !ok { + return fmt.Errorf("invalid type for service default: %T", resp.Entry.GetName()) + } + + snap.ConnectProxy.DestinationsUpstream.Set(uid, serviceConf) + case strings.HasPrefix(u.CorrelationID, DestinationGatewayID): + resp, ok := u.Result.(*structs.IndexedCheckServiceNodes) + if !ok { + return fmt.Errorf("invalid type for response: %T", u.Result) + } + + pq := strings.TrimPrefix(u.CorrelationID, DestinationGatewayID) + uid := UpstreamIDFromString(pq) + snap.ConnectProxy.DestinationGateways.Set(uid, resp.Nodes) case strings.HasPrefix(u.CorrelationID, "upstream:"+preparedQueryIDPrefix): resp, ok := u.Result.(*structs.PreparedQueryExecuteResponse) if !ok { diff --git a/agent/proxycfg/data_sources.go b/agent/proxycfg/data_sources.go index 310a4340ea..3bef5e3478 100644 --- a/agent/proxycfg/data_sources.go +++ b/agent/proxycfg/data_sources.go @@ -47,6 +47,10 @@ type DataSources struct { // notification channel. GatewayServices GatewayServices + // ServiceGateways provides updates about a gateway's upstream services on a + // notification channel. + ServiceGateways ServiceGateways + // Health provides service health updates on a notification channel. Health Health @@ -61,6 +65,10 @@ type DataSources struct { // notification channel. IntentionUpstreams IntentionUpstreams + // IntentionUpstreamsDestination provides intention-inferred upstream updates on a + // notification channel. + IntentionUpstreamsDestination IntentionUpstreamsDestination + // InternalServiceDump provides updates about a (gateway) service on a // notification channel. InternalServiceDump InternalServiceDump @@ -115,7 +123,7 @@ type ConfigEntry interface { Notify(ctx context.Context, req *structs.ConfigEntryQuery, correlationID string, ch chan<- UpdateEvent) error } -// ConfigEntry is the interface used to consume updates about a list of config +// ConfigEntryList is the interface used to consume updates about a list of config // entries. type ConfigEntryList interface { Notify(ctx context.Context, req *structs.ConfigEntryQuery, correlationID string, ch chan<- UpdateEvent) error @@ -139,6 +147,11 @@ type GatewayServices interface { Notify(ctx context.Context, req *structs.ServiceSpecificRequest, correlationID string, ch chan<- UpdateEvent) error } +// ServiceGateways is the interface used to consume updates about a service terminating gateways +type ServiceGateways interface { + Notify(ctx context.Context, req *structs.ServiceSpecificRequest, correlationID string, ch chan<- UpdateEvent) error +} + // Health is the interface used to consume service health updates. type Health interface { Notify(ctx context.Context, req *structs.ServiceSpecificRequest, correlationID string, ch chan<- UpdateEvent) error @@ -162,6 +175,12 @@ type IntentionUpstreams interface { Notify(ctx context.Context, req *structs.ServiceSpecificRequest, correlationID string, ch chan<- UpdateEvent) error } +// IntentionUpstreamsDestination is the interface used to consume updates about upstreams destination +// inferred from service intentions. +type IntentionUpstreamsDestination interface { + Notify(ctx context.Context, req *structs.ServiceSpecificRequest, correlationID string, ch chan<- UpdateEvent) error +} + // InternalServiceDump is the interface used to consume updates about a (gateway) // service via the internal ServiceDump RPC. type InternalServiceDump interface { diff --git a/agent/proxycfg/internal/watch/watchmap.go b/agent/proxycfg/internal/watch/watchmap.go index bbf42dc9af..ec676bb8f9 100644 --- a/agent/proxycfg/internal/watch/watchmap.go +++ b/agent/proxycfg/internal/watch/watchmap.go @@ -106,3 +106,18 @@ func (m Map[K, V]) ForEachKey(f func(K) bool) { } } } + +// ForEachKeyE iterates through the map, calling f +// for each iteration. It is up to the caller to +// Get the value and nil-check if required. +// If a non-nil error is returned by f, iterating +// stops and the error is returned. +// Order of iteration is non-deterministic. +func (m Map[K, V]) ForEachKeyE(f func(K) error) error { + for k := range m.M { + if err := f(k); err != nil { + return err + } + } + return nil +} diff --git a/agent/proxycfg/internal/watch/watchmap_test.go b/agent/proxycfg/internal/watch/watchmap_test.go index 590351853e..deb7cea08a 100644 --- a/agent/proxycfg/internal/watch/watchmap_test.go +++ b/agent/proxycfg/internal/watch/watchmap_test.go @@ -1,6 +1,7 @@ package watch import ( + "errors" "testing" "github.com/stretchr/testify/require" @@ -111,3 +112,43 @@ func TestMap_ForEach(t *testing.T) { require.Equal(t, 1, count) } } + +func TestMap_ForEachE(t *testing.T) { + type testType struct { + s string + } + + m := NewMap[string, any]() + inputs := map[string]any{ + "hello": 13, + "foo": struct{}{}, + "bar": &testType{s: "wow"}, + } + for k, v := range inputs { + m.InitWatch(k, nil) + m.Set(k, v) + } + require.Equal(t, 3, m.Len()) + + // returning nil error continues iteration + { + var count int + err := m.ForEachKeyE(func(k string) error { + count++ + return nil + }) + require.Equal(t, 3, count) + require.Nil(t, err) + } + + // returning an error should exit immediately + { + var count int + err := m.ForEachKeyE(func(k string) error { + count++ + return errors.New("boooo") + }) + require.Equal(t, 1, count) + require.Errorf(t, err, "boo") + } +} diff --git a/agent/proxycfg/manager_test.go b/agent/proxycfg/manager_test.go index 184b62148f..2a3cdd15f2 100644 --- a/agent/proxycfg/manager_test.go +++ b/agent/proxycfg/manager_test.go @@ -236,6 +236,8 @@ func TestManager_BasicLifecycle(t *testing.T) { PeerUpstreamEndpointsUseHostnames: map[UpstreamID]struct{}{}, }, PreparedQueryEndpoints: map[UpstreamID]structs.CheckServiceNodes{}, + DestinationsUpstream: watch.NewMap[UpstreamID, *structs.ServiceConfigEntry](), + DestinationGateways: watch.NewMap[UpstreamID, structs.CheckServiceNodes](), WatchedServiceChecks: map[structs.ServiceID][]structs.CheckType{}, Intentions: TestIntentions(), IntentionsSet: true, @@ -297,6 +299,8 @@ func TestManager_BasicLifecycle(t *testing.T) { PeerUpstreamEndpointsUseHostnames: map[UpstreamID]struct{}{}, }, PreparedQueryEndpoints: map[UpstreamID]structs.CheckServiceNodes{}, + DestinationsUpstream: watch.NewMap[UpstreamID, *structs.ServiceConfigEntry](), + DestinationGateways: watch.NewMap[UpstreamID, structs.CheckServiceNodes](), WatchedServiceChecks: map[structs.ServiceID][]structs.CheckType{}, Intentions: TestIntentions(), IntentionsSet: true, diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 6a02aad1e6..b04c67c26c 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -142,6 +142,9 @@ type configSnapshotConnectProxy struct { // intentions. Intentions structs.Intentions IntentionsSet bool + + DestinationsUpstream watch.Map[UpstreamID, *structs.ServiceConfigEntry] + DestinationGateways watch.Map[UpstreamID, structs.CheckServiceNodes] } // isEmpty is a test helper @@ -163,6 +166,8 @@ func (c *configSnapshotConnectProxy) isEmpty() bool { len(c.UpstreamConfig) == 0 && len(c.PassthroughUpstreams) == 0 && len(c.IntentionUpstreams) == 0 && + c.DestinationGateways.Len() == 0 && + c.DestinationsUpstream.Len() == 0 && len(c.PeeredUpstreams) == 0 && !c.InboundPeerTrustBundlesSet && !c.MeshConfigSet && diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index f9388cf487..13b22c4fd2 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -37,9 +37,12 @@ const ( serviceIntentionsIDPrefix = "service-intentions:" intentionUpstreamsID = "intention-upstreams" peeredUpstreamsID = "peered-upstreams" + intentionUpstreamsDestinationID = "intention-upstreams-destination" upstreamPeerWatchIDPrefix = "upstream-peer:" exportedServiceListWatchID = "exported-service-list" meshConfigEntryID = "mesh" + DestinationConfigEntryID = "destination:" + DestinationGatewayID = "dest-gateway:" svcChecksWatchIDPrefix = cachetype.ServiceHTTPChecksName + ":" preparedQueryIDPrefix = string(structs.UpstreamDestTypePreparedQuery) + ":" defaultPreparedQueryPollInterval = 30 * time.Second diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 36b641a691..662596b9be 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -125,10 +125,12 @@ func recordWatches(sc *stateConfig) *watchRecorder { Datacenters: typedWatchRecorder[*structs.DatacentersRequest]{wr}, FederationStateListMeshGateways: typedWatchRecorder[*structs.DCSpecificRequest]{wr}, GatewayServices: typedWatchRecorder[*structs.ServiceSpecificRequest]{wr}, + ServiceGateways: typedWatchRecorder[*structs.ServiceSpecificRequest]{wr}, Health: typedWatchRecorder[*structs.ServiceSpecificRequest]{wr}, HTTPChecks: typedWatchRecorder[*cachetype.ServiceHTTPChecksRequest]{wr}, Intentions: typedWatchRecorder[*structs.ServiceSpecificRequest]{wr}, IntentionUpstreams: typedWatchRecorder[*structs.ServiceSpecificRequest]{wr}, + IntentionUpstreamsDestination: typedWatchRecorder[*structs.ServiceSpecificRequest]{wr}, InternalServiceDump: typedWatchRecorder[*structs.ServiceDumpRequest]{wr}, LeafCertificate: typedWatchRecorder[*cachetype.ConnectCALeafRequest]{wr}, PeeredUpstreams: typedWatchRecorder[*structs.PartitionSpecificRequest]{wr}, @@ -1738,11 +1740,12 @@ func TestState_WatchesAndUpdates(t *testing.T) { stages: []verificationStage{ { requiredWatches: map[string]verifyWatchRequest{ - intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), - intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), - meshConfigEntryID: genVerifyMeshConfigWatch("dc1"), - rootsWatchID: genVerifyDCSpecificWatch("dc1"), - leafWatchID: genVerifyLeafWatch("api", "dc1"), + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + meshConfigEntryID: genVerifyMeshConfigWatch("dc1"), + rootsWatchID: genVerifyDCSpecificWatch("dc1"), + leafWatchID: genVerifyLeafWatch("api", "dc1"), }, verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { require.False(t, snap.Valid(), "proxy without roots/leaf/intentions is not valid") @@ -1823,11 +1826,12 @@ func TestState_WatchesAndUpdates(t *testing.T) { // Empty on initialization { requiredWatches: map[string]verifyWatchRequest{ - intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), - intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), - meshConfigEntryID: genVerifyMeshConfigWatch("dc1"), - rootsWatchID: genVerifyDCSpecificWatch("dc1"), - leafWatchID: genVerifyLeafWatch("api", "dc1"), + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + meshConfigEntryID: genVerifyMeshConfigWatch("dc1"), + rootsWatchID: genVerifyDCSpecificWatch("dc1"), + leafWatchID: genVerifyLeafWatch("api", "dc1"), }, verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { require.False(t, snap.Valid(), "proxy without roots/leaf/intentions is not valid") @@ -1882,10 +1886,11 @@ func TestState_WatchesAndUpdates(t *testing.T) { // Receiving an intention should lead to spinning up a discovery chain watch { requiredWatches: map[string]verifyWatchRequest{ - intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), - intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), - rootsWatchID: genVerifyDCSpecificWatch("dc1"), - leafWatchID: genVerifyLeafWatch("api", "dc1"), + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + rootsWatchID: genVerifyDCSpecificWatch("dc1"), + leafWatchID: genVerifyLeafWatch("api", "dc1"), }, events: []UpdateEvent{ { @@ -2313,10 +2318,11 @@ func TestState_WatchesAndUpdates(t *testing.T) { { // Empty list of upstreams should clean up map keys requiredWatches: map[string]verifyWatchRequest{ - intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), - intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), - rootsWatchID: genVerifyDCSpecificWatch("dc1"), - leafWatchID: genVerifyLeafWatch("api", "dc1"), + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + rootsWatchID: genVerifyDCSpecificWatch("dc1"), + leafWatchID: genVerifyLeafWatch("api", "dc1"), }, events: []UpdateEvent{ { @@ -2344,6 +2350,169 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, }, }, + "transparent-proxy-handle-update-destination": { + ns: structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + ID: "api-proxy", + Service: "api-proxy", + Address: "10.0.1.1", + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "api", + Mode: structs.ProxyModeTransparent, + Upstreams: structs.Upstreams{ + { + CentrallyConfigured: true, + DestinationName: structs.WildcardSpecifier, + DestinationNamespace: structs.WildcardSpecifier, + Config: map[string]interface{}{ + "connect_timeout_ms": 6000, + }, + MeshGateway: structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeRemote}, + }, + }, + }, + }, + sourceDC: "dc1", + stages: []verificationStage{ + // Empty on initialization + { + requiredWatches: map[string]verifyWatchRequest{ + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + meshConfigEntryID: genVerifyMeshConfigWatch("dc1"), + rootsWatchID: genVerifyDCSpecificWatch("dc1"), + leafWatchID: genVerifyLeafWatch("api", "dc1"), + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.False(t, snap.Valid(), "proxy without roots/leaf/intentions is not valid") + require.True(t, snap.MeshGateway.isEmpty()) + require.True(t, snap.IngressGateway.isEmpty()) + require.True(t, snap.TerminatingGateway.isEmpty()) + + // Centrally configured upstream defaults should be stored so that upstreams from intentions can inherit them + require.Len(t, snap.ConnectProxy.UpstreamConfig, 1) + + wc := structs.NewServiceName(structs.WildcardSpecifier, structs.WildcardEnterpriseMetaInDefaultPartition()) + wcUID := NewUpstreamIDFromServiceName(wc) + require.Contains(t, snap.ConnectProxy.UpstreamConfig, wcUID) + }, + }, + // Valid snapshot after roots, leaf, and intentions + { + events: []UpdateEvent{ + rootWatchEvent(), + { + CorrelationID: leafWatchID, + Result: issuedCert, + Err: nil, + }, + { + CorrelationID: intentionsWatchID, + Result: TestIntentions(), + Err: nil, + }, + { + CorrelationID: meshConfigEntryID, + Result: &structs.ConfigEntryResponse{ + Entry: &structs.MeshConfigEntry{ + TransparentProxy: structs.TransparentProxyMeshConfig{}, + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "proxy with roots/leaf/intentions is valid") + require.Equal(t, indexedRoots, snap.Roots) + require.Equal(t, issuedCert, snap.Leaf()) + require.Equal(t, TestIntentions(), snap.ConnectProxy.Intentions) + require.True(t, snap.MeshGateway.isEmpty()) + require.True(t, snap.IngressGateway.isEmpty()) + require.True(t, snap.TerminatingGateway.isEmpty()) + require.True(t, snap.ConnectProxy.MeshConfigSet) + require.NotNil(t, snap.ConnectProxy.MeshConfig) + }, + }, + // Receiving an intention should lead to spinning up a DestinationConfigEntryID + { + requiredWatches: map[string]verifyWatchRequest{ + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + rootsWatchID: genVerifyDCSpecificWatch("dc1"), + leafWatchID: genVerifyLeafWatch("api", "dc1"), + }, + events: []UpdateEvent{ + { + CorrelationID: intentionUpstreamsDestinationID, + Result: &structs.IndexedServiceList{ + Services: structs.ServiceList{ + db, + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "should still be valid") + + // Watches have a key allocated even if the value is not set + require.Equal(t, 1, snap.ConnectProxy.DestinationsUpstream.Len()) + }, + }, + // DestinationConfigEntryID updates should be stored + { + requiredWatches: map[string]verifyWatchRequest{ + DestinationConfigEntryID + dbUID.String(): genVerifyConfigEntryWatch(structs.ServiceDefaults, db.Name, "dc1"), + }, + events: []UpdateEvent{ + { + CorrelationID: DestinationConfigEntryID + dbUID.String(), + Result: &structs.ConfigEntryResponse{ + Entry: &structs.ServiceConfigEntry{Name: "db", Destination: &structs.DestinationConfig{}}, + }, + Err: nil, + }, + { + CorrelationID: DestinationGatewayID + dbUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "foo", + Partition: api.PartitionOrDefault(), + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "gtwy1", + TaggedAddresses: map[string]structs.ServiceAddress{ + structs.ServiceGatewayVirtualIPTag(structs.ServiceName{Name: "db", EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition()}): {Address: "172.0.0.1", Port: 443}, + }, + }, + Checks: structs.HealthChecks{}, + }, + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid(), "should still be valid") + require.Equal(t, 1, snap.ConnectProxy.DestinationsUpstream.Len()) + require.Equal(t, 1, snap.ConnectProxy.DestinationGateways.Len()) + snap.ConnectProxy.DestinationsUpstream.ForEachKey(func(uid UpstreamID) bool { + _, ok := snap.ConnectProxy.DestinationsUpstream.Get(uid) + require.True(t, ok) + return true + }) + dbDest, ok := snap.ConnectProxy.DestinationsUpstream.Get(dbUID) + require.True(t, ok) + require.Equal(t, structs.ServiceConfigEntry{Name: "db", Destination: &structs.DestinationConfig{}}, *dbDest) + }, + }, + }, + }, // Receiving an empty upstreams from Intentions list shouldn't delete explicit upstream watches "transparent-proxy-handle-update-explicit-cross-dc": { ns: structs.NodeService{ @@ -2379,9 +2548,10 @@ func TestState_WatchesAndUpdates(t *testing.T) { // Empty on initialization { requiredWatches: map[string]verifyWatchRequest{ - intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), - intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), - meshConfigEntryID: genVerifyMeshConfigWatch("dc1"), + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + meshConfigEntryID: genVerifyMeshConfigWatch("dc1"), "discovery-chain:" + upstreamIDForDC2(dbUID).String(): genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ Name: "db", EvaluateInDatacenter: "dc2", @@ -2479,8 +2649,9 @@ func TestState_WatchesAndUpdates(t *testing.T) { // be deleted from the snapshot. { requiredWatches: map[string]verifyWatchRequest{ - intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), - intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionsWatchID: genVerifyIntentionWatch("api", "dc1"), + intentionUpstreamsID: genVerifyServiceSpecificRequest("api", "", "dc1", false), + intentionUpstreamsDestinationID: genVerifyServiceSpecificRequest("api", "", "dc1", false), "discovery-chain:" + upstreamIDForDC2(dbUID).String(): genVerifyDiscoveryChainWatch(&structs.DiscoveryChainRequest{ Name: "db", EvaluateInDatacenter: "dc2", diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 744c17e182..dfde519d18 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -739,10 +739,12 @@ func testConfigSnapshotFixture( Datacenters: &noopDataSource[*structs.DatacentersRequest]{}, FederationStateListMeshGateways: &noopDataSource[*structs.DCSpecificRequest]{}, GatewayServices: &noopDataSource[*structs.ServiceSpecificRequest]{}, + ServiceGateways: &noopDataSource[*structs.ServiceSpecificRequest]{}, Health: &noopDataSource[*structs.ServiceSpecificRequest]{}, HTTPChecks: &noopDataSource[*cachetype.ServiceHTTPChecksRequest]{}, Intentions: &noopDataSource[*structs.ServiceSpecificRequest]{}, IntentionUpstreams: &noopDataSource[*structs.ServiceSpecificRequest]{}, + IntentionUpstreamsDestination: &noopDataSource[*structs.ServiceSpecificRequest]{}, InternalServiceDump: &noopDataSource[*structs.ServiceDumpRequest]{}, LeafCertificate: &noopDataSource[*cachetype.ConnectCALeafRequest]{}, PeeredUpstreams: &noopDataSource[*structs.PartitionSpecificRequest]{}, @@ -946,6 +948,7 @@ func NewTestDataSources() *TestDataSources { HTTPChecks: NewTestDataSource[*cachetype.ServiceHTTPChecksRequest, []structs.CheckType](), Intentions: NewTestDataSource[*structs.ServiceSpecificRequest, structs.Intentions](), IntentionUpstreams: NewTestDataSource[*structs.ServiceSpecificRequest, *structs.IndexedServiceList](), + IntentionUpstreamsDestination: NewTestDataSource[*structs.ServiceSpecificRequest, *structs.IndexedServiceList](), InternalServiceDump: NewTestDataSource[*structs.ServiceDumpRequest, *structs.IndexedNodesWithGateways](), LeafCertificate: NewTestDataSource[*cachetype.ConnectCALeafRequest, *structs.IssuedCert](), PreparedQuery: NewTestDataSource[*structs.PreparedQueryExecuteRequest, *structs.PreparedQueryExecuteResponse](), @@ -966,10 +969,12 @@ type TestDataSources struct { FederationStateListMeshGateways *TestDataSource[*structs.DCSpecificRequest, *structs.DatacenterIndexedCheckServiceNodes] Datacenters *TestDataSource[*structs.DatacentersRequest, *[]string] GatewayServices *TestDataSource[*structs.ServiceSpecificRequest, *structs.IndexedGatewayServices] + ServiceGateways *TestDataSource[*structs.ServiceSpecificRequest, *structs.IndexedServiceNodes] Health *TestDataSource[*structs.ServiceSpecificRequest, *structs.IndexedCheckServiceNodes] HTTPChecks *TestDataSource[*cachetype.ServiceHTTPChecksRequest, []structs.CheckType] Intentions *TestDataSource[*structs.ServiceSpecificRequest, structs.Intentions] IntentionUpstreams *TestDataSource[*structs.ServiceSpecificRequest, *structs.IndexedServiceList] + IntentionUpstreamsDestination *TestDataSource[*structs.ServiceSpecificRequest, *structs.IndexedServiceList] InternalServiceDump *TestDataSource[*structs.ServiceDumpRequest, *structs.IndexedNodesWithGateways] LeafCertificate *TestDataSource[*cachetype.ConnectCALeafRequest, *structs.IssuedCert] PeeredUpstreams *TestDataSource[*structs.PartitionSpecificRequest, *structs.IndexedPeeredServiceList] @@ -984,24 +989,26 @@ type TestDataSources struct { func (t *TestDataSources) ToDataSources() DataSources { ds := DataSources{ - CARoots: t.CARoots, - CompiledDiscoveryChain: t.CompiledDiscoveryChain, - ConfigEntry: t.ConfigEntry, - ConfigEntryList: t.ConfigEntryList, - Datacenters: t.Datacenters, - GatewayServices: t.GatewayServices, - Health: t.Health, - HTTPChecks: t.HTTPChecks, - Intentions: t.Intentions, - IntentionUpstreams: t.IntentionUpstreams, - InternalServiceDump: t.InternalServiceDump, - LeafCertificate: t.LeafCertificate, - PeeredUpstreams: t.PeeredUpstreams, - PreparedQuery: t.PreparedQuery, - ResolvedServiceConfig: t.ResolvedServiceConfig, - ServiceList: t.ServiceList, - TrustBundle: t.TrustBundle, - TrustBundleList: t.TrustBundleList, + CARoots: t.CARoots, + CompiledDiscoveryChain: t.CompiledDiscoveryChain, + ConfigEntry: t.ConfigEntry, + ConfigEntryList: t.ConfigEntryList, + Datacenters: t.Datacenters, + GatewayServices: t.GatewayServices, + ServiceGateways: t.ServiceGateways, + Health: t.Health, + HTTPChecks: t.HTTPChecks, + Intentions: t.Intentions, + IntentionUpstreams: t.IntentionUpstreams, + IntentionUpstreamsDestination: t.IntentionUpstreamsDestination, + InternalServiceDump: t.InternalServiceDump, + LeafCertificate: t.LeafCertificate, + PeeredUpstreams: t.PeeredUpstreams, + PreparedQuery: t.PreparedQuery, + ResolvedServiceConfig: t.ResolvedServiceConfig, + ServiceList: t.ServiceList, + TrustBundle: t.TrustBundle, + TrustBundleList: t.TrustBundleList, } t.fillEnterpriseDataSources(&ds) return ds diff --git a/agent/proxycfg/testing_terminating_gateway.go b/agent/proxycfg/testing_terminating_gateway.go index 00771433b5..64a624e70f 100644 --- a/agent/proxycfg/testing_terminating_gateway.go +++ b/agent/proxycfg/testing_terminating_gateway.go @@ -328,8 +328,10 @@ func TestConfigSnapshotTerminatingGatewayDestinations(t testing.T, populateDesti roots, _ := TestCerts(t) var ( - externalIPTCP = structs.NewServiceName("external-IP-TCP", nil) - externalHostnameTCP = structs.NewServiceName("external-hostname-TCP", nil) + externalIPTCP = structs.NewServiceName("external-IP-TCP", nil) + externalHostnameTCP = structs.NewServiceName("external-hostname-TCP", nil) + externalIPHTTP = structs.NewServiceName("external-IP-HTTP", nil) + externalHostnameHTTP = structs.NewServiceName("external-hostname-HTTP", nil) ) baseEvents := []UpdateEvent{ @@ -357,6 +359,14 @@ func TestConfigSnapshotTerminatingGatewayDestinations(t testing.T, populateDesti Service: externalHostnameTCP, ServiceKind: structs.GatewayServiceKindDestination, }, + &structs.GatewayService{ + Service: externalIPHTTP, + ServiceKind: structs.GatewayServiceKindDestination, + }, + &structs.GatewayService{ + Service: externalHostnameHTTP, + ServiceKind: structs.GatewayServiceKindDestination, + }, ) baseEvents = testSpliceEvents(baseEvents, []UpdateEvent{ @@ -375,6 +385,14 @@ func TestConfigSnapshotTerminatingGatewayDestinations(t testing.T, populateDesti CorrelationID: serviceIntentionsIDPrefix + externalHostnameTCP.String(), Result: structs.Intentions{}, }, + { + CorrelationID: serviceIntentionsIDPrefix + externalIPHTTP.String(), + Result: structs.Intentions{}, + }, + { + CorrelationID: serviceIntentionsIDPrefix + externalHostnameHTTP.String(), + Result: structs.Intentions{}, + }, // ======== { CorrelationID: serviceLeafIDPrefix + externalIPTCP.String(), @@ -390,6 +408,20 @@ func TestConfigSnapshotTerminatingGatewayDestinations(t testing.T, populateDesti PrivateKeyPEM: "placeholder.key", }, }, + { + CorrelationID: serviceLeafIDPrefix + externalIPHTTP.String(), + Result: &structs.IssuedCert{ + CertPEM: "placeholder.crt", + PrivateKeyPEM: "placeholder.key", + }, + }, + { + CorrelationID: serviceLeafIDPrefix + externalHostnameHTTP.String(), + Result: &structs.IssuedCert{ + CertPEM: "placeholder.crt", + PrivateKeyPEM: "placeholder.key", + }, + }, // ======== { CorrelationID: serviceConfigIDPrefix + externalIPTCP.String(), @@ -408,11 +440,33 @@ func TestConfigSnapshotTerminatingGatewayDestinations(t testing.T, populateDesti Mode: structs.ProxyModeTransparent, ProxyConfig: map[string]interface{}{"protocol": "tcp"}, Destination: structs.DestinationConfig{ - Address: "*.hashicorp.com", + Address: "api.hashicorp.com", Port: 8089, }, }, }, + { + CorrelationID: serviceConfigIDPrefix + externalIPHTTP.String(), + Result: &structs.ServiceConfigResponse{ + Mode: structs.ProxyModeTransparent, + ProxyConfig: map[string]interface{}{"protocol": "http"}, + Destination: structs.DestinationConfig{ + Address: "192.168.0.2", + Port: 80, + }, + }, + }, + { + CorrelationID: serviceConfigIDPrefix + externalHostnameHTTP.String(), + Result: &structs.ServiceConfigResponse{ + Mode: structs.ProxyModeTransparent, + ProxyConfig: map[string]interface{}{"protocol": "http"}, + Destination: structs.DestinationConfig{ + Address: "httpbin.org", + Port: 80, + }, + }, + }, }) } diff --git a/agent/proxycfg/testing_tproxy.go b/agent/proxycfg/testing_tproxy.go index b93e6c970b..ab55f3313f 100644 --- a/agent/proxycfg/testing_tproxy.go +++ b/agent/proxycfg/testing_tproxy.go @@ -1,6 +1,7 @@ package proxycfg import ( + "github.com/hashicorp/consul/api" "time" "github.com/mitchellh/go-testing-interface" @@ -522,3 +523,117 @@ func TestConfigSnapshotTransparentProxyTerminatingGatewayCatalogDestinationsOnly }, }) } + +func TestConfigSnapshotTransparentProxyDestination(t testing.T) *ConfigSnapshot { + // DiscoveryChain without an UpstreamConfig should yield a + // filter chain when in transparent proxy mode + var ( + google = structs.NewServiceName("google", nil) + googleUID = NewUpstreamIDFromServiceName(google) + googleCE = structs.ServiceConfigEntry{Name: "google", Destination: &structs.DestinationConfig{Address: "www.google.com", Port: 443}} + + kafka = structs.NewServiceName("kafka", nil) + kafkaUID = NewUpstreamIDFromServiceName(kafka) + kafkaCE = structs.ServiceConfigEntry{Name: "kafka", Destination: &structs.DestinationConfig{Address: "192.168.2.1", Port: 9093}} + ) + + return TestConfigSnapshot(t, func(ns *structs.NodeService) { + ns.Proxy.Mode = structs.ProxyModeTransparent + }, []UpdateEvent{ + { + CorrelationID: meshConfigEntryID, + Result: &structs.ConfigEntryResponse{ + Entry: &structs.MeshConfigEntry{ + TransparentProxy: structs.TransparentProxyMeshConfig{ + MeshDestinationsOnly: true, + }, + }, + }, + }, + { + CorrelationID: intentionUpstreamsDestinationID, + Result: &structs.IndexedServiceList{ + Services: structs.ServiceList{ + google, + kafka, + }, + }, + }, + { + CorrelationID: DestinationConfigEntryID + googleUID.String(), + Result: &structs.ConfigEntryResponse{ + Entry: &googleCE, + }, + }, + { + CorrelationID: DestinationConfigEntryID + kafkaUID.String(), + Result: &structs.ConfigEntryResponse{ + Entry: &kafkaCE, + }, + }, + { + CorrelationID: DestinationGatewayID + googleUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "172.168.0.1", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + ID: "tgtw1", + Address: "172.168.0.1", + Port: 8443, + Kind: structs.ServiceKindTerminatingGateway, + TaggedAddresses: map[string]structs.ServiceAddress{ + structs.TaggedAddressLANIPv4: {Address: "172.168.0.1", Port: 8443}, + structs.TaggedAddressVirtualIP: {Address: "240.0.0.1"}, + }, + }, + Checks: []*structs.HealthCheck{ + { + Node: "node1", + ServiceName: "tgtw", + Name: "force", + Status: api.HealthPassing, + }, + }, + }, + }, + }, + }, + { + CorrelationID: DestinationGatewayID + kafkaUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: structs.CheckServiceNodes{ + { + Node: &structs.Node{ + Node: "node1", + Address: "172.168.0.1", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + ID: "tgtw1", + Address: "172.168.0.1", + Port: 8443, + Kind: structs.ServiceKindTerminatingGateway, + TaggedAddresses: map[string]structs.ServiceAddress{ + structs.TaggedAddressLANIPv4: {Address: "172.168.0.1", Port: 8443}, + structs.TaggedAddressVirtualIP: {Address: "240.0.0.1"}, + }, + }, + Checks: []*structs.HealthCheck{ + { + Node: "node1", + ServiceName: "tgtw", + Name: "force", + Status: api.HealthPassing, + }, + }, + }, + }, + }, + }, + }) +} diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index b4f4eea39b..562e7e6921 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -9,8 +9,6 @@ import ( envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" - envoy_cluster_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3" - envoy_common_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/common/dynamic_forward_proxy/v3" envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" envoy_upstreams_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" envoy_matcher_v3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" @@ -29,12 +27,6 @@ import ( "github.com/hashicorp/consul/agent/structs" ) -const ( - dynamicForwardProxyClusterName = "dynamic_forward_proxy_cluster" - dynamicForwardProxyClusterTypeName = "envoy.clusters.dynamic_forward_proxy" - dynamicForwardProxyClusterDNSCacheName = "dynamic_forward_proxy_cache_config" -) - const ( meshGatewayExportedClusterNamePrefix = "exported~" ) @@ -247,28 +239,7 @@ func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, c.ConnectTimeout = durationpb.New(discoTarget.ConnectTimeout) } - spiffeID := connect.SpiffeIDService{ - Host: cfgSnap.Roots.TrustDomain, - Partition: uid.PartitionOrDefault(), - Namespace: uid.NamespaceOrDefault(), - Datacenter: cfgSnap.Datacenter, - Service: uid.Name, - } - - commonTLSContext := makeCommonTLSContext( - cfgSnap.Leaf(), - cfgSnap.RootPEMs(), - makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()), - ) - err := injectSANMatcher(commonTLSContext, spiffeID.URI().String()) - if err != nil { - return nil, fmt.Errorf("failed to inject SAN matcher rules for cluster %q: %v", sni, err) - } - tlsContext := envoy_tls_v3.UpstreamTlsContext{ - CommonTlsContext: commonTLSContext, - Sni: sni, - } - transportSocket, err := makeUpstreamTLSTransportSocket(&tlsContext) + transportSocket, err := makeMTLSTransportSocket(cfgSnap, uid, sni) if err != nil { return nil, err } @@ -277,9 +248,84 @@ func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, } } + err := cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKeyE(func(uid proxycfg.UpstreamID) error { + name := clusterNameForDestination(cfgSnap, uid.Name, uid.NamespaceOrDefault(), uid.PartitionOrDefault()) + + c := envoy_cluster_v3.Cluster{ + Name: name, + AltStatName: name, + ConnectTimeout: durationpb.New(5 * time.Second), + CommonLbConfig: &envoy_cluster_v3.Cluster_CommonLbConfig{ + HealthyPanicThreshold: &envoy_type_v3.Percent{ + Value: 0, // disable panic threshold + }, + }, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}, + EdsClusterConfig: &envoy_cluster_v3.Cluster_EdsClusterConfig{ + EdsConfig: &envoy_core_v3.ConfigSource{ + ResourceApiVersion: envoy_core_v3.ApiVersion_V3, + ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{ + Ads: &envoy_core_v3.AggregatedConfigSource{}, + }, + }, + }, + // Endpoints are managed separately by EDS + // Having an empty config enables outlier detection with default config. + OutlierDetection: &envoy_cluster_v3.OutlierDetection{}, + } + + // Use the cluster name as the SNI to match on in the terminating gateway + transportSocket, err := makeMTLSTransportSocket(cfgSnap, uid, name) + if err != nil { + return err + } + c.TransportSocket = transportSocket + clusters = append(clusters, &c) + return nil + }) + if err != nil { + return nil, err + } + return clusters, nil } +func makeMTLSTransportSocket(cfgSnap *proxycfg.ConfigSnapshot, uid proxycfg.UpstreamID, sni string) (*envoy_core_v3.TransportSocket, error) { + spiffeID := connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Partition: uid.PartitionOrDefault(), + Namespace: uid.NamespaceOrDefault(), + Datacenter: cfgSnap.Datacenter, + Service: uid.Name, + } + + commonTLSContext := makeCommonTLSContext( + cfgSnap.Leaf(), + cfgSnap.RootPEMs(), + makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()), + ) + err := injectSANMatcher(commonTLSContext, spiffeID.URI().String()) + if err != nil { + return nil, fmt.Errorf("failed to inject SAN matcher rules for cluster %q: %v", sni, err) + } + tlsContext := envoy_tls_v3.UpstreamTlsContext{ + CommonTlsContext: commonTLSContext, + Sni: sni, + } + transportSocket, err := makeUpstreamTLSTransportSocket(&tlsContext) + if err != nil { + return nil, err + } + return transportSocket, nil +} + +func clusterNameForDestination(cfgSnap *proxycfg.ConfigSnapshot, name string, namespace string, partition string) string { + sni := connect.ServiceSNI(name, "", namespace, partition, cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) + + // Prefixed with destination to distinguish from non-passthrough clusters for the same upstream. + return "destination~" + sni +} + // clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters" // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. @@ -475,7 +521,6 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( } func (s *ResourceGenerator) makeDestinationClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - var createDynamicForwardProxy bool serviceConfigs := cfgSnap.TerminatingGateway.ServiceConfigs clusters := make([]proto.Message, 0, len(cfgSnap.TerminatingGateway.DestinationServices)) @@ -484,31 +529,17 @@ func (s *ResourceGenerator) makeDestinationClusters(cfgSnap *proxycfg.ConfigSnap svcConfig, _ := serviceConfigs[svcName] dest := svcConfig.Destination - // If IP, create a cluster with the fake name. - if dest.HasIP() { - opts := clusterOpts{ - name: connect.ServiceSNI(svcName.Name, "", svcName.NamespaceOrDefault(), svcName.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain), - addressEndpoint: dest, - } - cluster := s.makeTerminatingIPCluster(cfgSnap, opts) - clusters = append(clusters, cluster) - continue - } - - // TODO (dans): clusters will need to be customized later when we figure out how to manage a TLS segment from the terminating gateway to the Destination. - createDynamicForwardProxy = true - } - - if createDynamicForwardProxy { opts := clusterOpts{ - name: dynamicForwardProxyClusterName, + name: clusterNameForDestination(cfgSnap, svcName.Name, svcName.NamespaceOrDefault(), svcName.PartitionOrDefault()), + addressEndpoint: dest, } - cluster := s.makeDynamicForwardProxyCluster(cfgSnap, opts) - // TODO (dans): might be relevant later for TLS addons like CA validation - // if err := s.injectGatewayServiceAddons(cfgSnap, cluster, svc, loadBalancer); err != nil { - // return nil, err - // } + var cluster *envoy_cluster_v3.Cluster + if dest.HasIP() { + cluster = s.makeTerminatingIPCluster(cfgSnap, opts) + } else { + cluster = s.makeTerminatingHostnameCluster(cfgSnap, opts) + } clusters = append(clusters, cluster) } return clusters, nil @@ -1360,7 +1391,7 @@ func configureClusterWithHostnames( } } -// makeGatewayCluster creates an Envoy cluster for a mesh or terminating gateway +// makeTerminatingIPCluster creates an Envoy cluster for a terminating gateway with an ip destination func (s *ResourceGenerator) makeTerminatingIPCluster(snap *proxycfg.ConfigSnapshot, opts clusterOpts) *envoy_cluster_v3.Cluster { cfg, err := ParseGatewayConfig(snap.Proxy.Config) if err != nil { @@ -1377,12 +1408,10 @@ func (s *ResourceGenerator) makeTerminatingIPCluster(snap *proxycfg.ConfigSnapsh ConnectTimeout: durationpb.New(opts.connectTimeout), // Having an empty config enables outlier detection with default config. - OutlierDetection: &envoy_cluster_v3.OutlierDetection{}, + OutlierDetection: &envoy_cluster_v3.OutlierDetection{}, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC}, } - discoveryType := envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC} - cluster.ClusterDiscoveryType = &discoveryType - endpoints := []*envoy_endpoint_v3.LbEndpoint{ makeEndpoint(opts.addressEndpoint.Address, opts.addressEndpoint.Port), } @@ -1398,47 +1427,49 @@ func (s *ResourceGenerator) makeTerminatingIPCluster(snap *proxycfg.ConfigSnapsh return cluster } -// makeDynamicForwardProxyCluster creates an Envoy cluster for that routes based on the SNI header received at the listener -func (s *ResourceGenerator) makeDynamicForwardProxyCluster(snap *proxycfg.ConfigSnapshot, opts clusterOpts) *envoy_cluster_v3.Cluster { +// makeTerminatingHostnameCluster creates an Envoy cluster for a terminating gateway with a hostname destination +func (s *ResourceGenerator) makeTerminatingHostnameCluster(snap *proxycfg.ConfigSnapshot, opts clusterOpts) *envoy_cluster_v3.Cluster { cfg, err := ParseGatewayConfig(snap.Proxy.Config) if err != nil { // Don't hard fail on a config typo, just warn. The parse func returns // default config if there is an error so it's safe to continue. s.Logger.Warn("failed to parse gateway config", "error", err) } - if opts.connectTimeout <= 0 { - opts.connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond - } + opts.connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond cluster := &envoy_cluster_v3.Cluster{ Name: opts.name, ConnectTimeout: durationpb.New(opts.connectTimeout), + + // Having an empty config enables outlier detection with default config. + OutlierDetection: &envoy_cluster_v3.OutlierDetection{}, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_LOGICAL_DNS}, + DnsLookupFamily: envoy_cluster_v3.Cluster_AUTO, } - dynamicForwardProxyCluster, err := anypb.New(&envoy_cluster_dynamic_forward_proxy_v3.ClusterConfig{ - DnsCacheConfig: getCommonDNSCacheConfiguration(), - }) - if err != nil { - // we should never get here since this message is static - s.Logger.Error("failed serialize dynamic forward proxy cluster config", "error", err) - } + rate := 10 * time.Second + cluster.DnsRefreshRate = durationpb.New(rate) - cluster.LbPolicy = envoy_cluster_v3.Cluster_CLUSTER_PROVIDED - cluster.ClusterDiscoveryType = &envoy_cluster_v3.Cluster_ClusterType{ - ClusterType: &envoy_cluster_v3.Cluster_CustomClusterType{ - Name: dynamicForwardProxyClusterTypeName, - TypedConfig: dynamicForwardProxyCluster, + address := makeAddress(opts.addressEndpoint.Address, opts.addressEndpoint.Port) + + endpoints := []*envoy_endpoint_v3.LbEndpoint{ + { + HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_endpoint_v3.Endpoint{ + Address: address, + }, + }, }, } - return cluster -} - -func getCommonDNSCacheConfiguration() *envoy_common_dynamic_forward_proxy_v3.DnsCacheConfig { - return &envoy_common_dynamic_forward_proxy_v3.DnsCacheConfig{ - Name: dynamicForwardProxyClusterDNSCacheName, - DnsLookupFamily: envoy_cluster_v3.Cluster_AUTO, + cluster.LoadAssignment = &envoy_endpoint_v3.ClusterLoadAssignment{ + ClusterName: cluster.Name, + Endpoints: []*envoy_endpoint_v3.LocalityLbEndpoints{{ + LbEndpoints: endpoints, + }}, } + + return cluster } func makeThresholdsIfNeeded(limits *structs.UpstreamLimits) []*envoy_cluster_v3.CircuitBreakers_Thresholds { diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 49d333750c..96e7615c70 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -621,12 +621,6 @@ func TestClustersFromSnapshot(t *testing.T) { name: "transparent-proxy-dial-instances-directly", create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, }, - { - name: "transparent-proxy-terminating-gateway-destinations-only", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotTerminatingGatewayDestinations(t, true, nil) - }, - }, } latestEnvoyVersion := proxysupport.EnvoyVersions[0] diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 2538914dd2..edfe1c6165 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -3,7 +3,6 @@ package xds import ( "errors" "fmt" - envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" @@ -149,6 +148,24 @@ func (s *ResourceGenerator) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg. } } + // Loop over potential destinations in the mesh, then grab the gateway nodes associated with each + cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKey(func(uid proxycfg.UpstreamID) bool { + name := clusterNameForDestination(cfgSnap, uid.Name, uid.NamespaceOrDefault(), uid.PartitionOrDefault()) + + endpoints, ok := cfgSnap.ConnectProxy.DestinationGateways.Get(uid) + if ok { + la := makeLoadAssignment( + name, + []loadAssignmentEndpointGroup{ + {Endpoints: endpoints}, + }, + proxycfg.GatewayKey{ /*empty so it never matches*/ }, + ) + resources = append(resources, la) + } + return true + }) + return resources, nil } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 2f3650aa77..0ef16899f5 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -22,7 +22,6 @@ import ( envoy_connection_limit_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/connection_limit/v3" envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" envoy_sni_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sni_cluster/v3" - envoy_sni_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3" envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" @@ -97,6 +96,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. outboundListener = makePortListener(OutboundListenerName, "127.0.0.1", port, envoy_core_v3.TrafficDirection_OUTBOUND) outboundListener.FilterChains = make([]*envoy_listener_v3.FilterChain, 0) + outboundListener.ListenerFilters = []*envoy_listener_v3.ListenerFilter{ // The original_dst filter is a listener filter that recovers the original destination // address before the iptables redirection. This filter is needed for transparent @@ -226,7 +226,44 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. outboundListener.FilterChains = append(outboundListener.FilterChains, filterChain) } } + hasDestination := false + err = cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKeyE(func(uid proxycfg.UpstreamID) error { + destination, ok := cfgSnap.ConnectProxy.DestinationsUpstream.Get(uid) + + if ok && destination != nil { + upstreamCfg := cfgSnap.ConnectProxy.UpstreamConfig[uid] + cfg := s.getAndModifyUpstreamConfigForListener(uid, upstreamCfg, nil) + + clusterName := clusterNameForDestination(cfgSnap, uid.Name, uid.NamespaceOrDefault(), uid.PartitionOrDefault()) + filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{ + routeName: uid.EnvoyID(), + clusterName: clusterName, + filterName: clusterName, + protocol: cfg.Protocol, + useRDS: cfg.Protocol != "tcp", + }) + if err != nil { + return err + } + filterChain.FilterChainMatch = makeFilterChainMatchFromAddressWithPort(destination.Destination.Address, destination.Destination.Port) + outboundListener.FilterChains = append(outboundListener.FilterChains, filterChain) + + hasDestination = len(filterChain.FilterChainMatch.ServerNames) != 0 || hasDestination + } + return nil + }) + if err != nil { + return nil, err + } + + if hasDestination { + tlsInspector, err := makeTLSInspectorListenerFilter() + if err != nil { + return nil, err + } + outboundListener.ListenerFilters = append(outboundListener.ListenerFilters, tlsInspector) + } // Looping over explicit upstreams is only needed for cross-peer because // they do not have discovery chains. for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { @@ -325,8 +362,27 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. // Filter chains are stable sorted to avoid draining if the list is provided out of order sort.SliceStable(outboundListener.FilterChains, func(i, j int) bool { - return outboundListener.FilterChains[i].FilterChainMatch.PrefixRanges[0].AddressPrefix < - outboundListener.FilterChains[j].FilterChainMatch.PrefixRanges[0].AddressPrefix + si := "" + sj := "" + if len(outboundListener.FilterChains[i].FilterChainMatch.PrefixRanges) > 0 { + si += outboundListener.FilterChains[i].FilterChainMatch.PrefixRanges[0].AddressPrefix + + "/" + outboundListener.FilterChains[i].FilterChainMatch.PrefixRanges[0].PrefixLen.String() + + ":" + outboundListener.FilterChains[i].FilterChainMatch.DestinationPort.String() + } + if len(outboundListener.FilterChains[i].FilterChainMatch.ServerNames) > 0 { + si += outboundListener.FilterChains[i].FilterChainMatch.ServerNames[0] + } + + if len(outboundListener.FilterChains[j].FilterChainMatch.PrefixRanges) > 0 { + sj += outboundListener.FilterChains[j].FilterChainMatch.PrefixRanges[0].AddressPrefix + + "/" + outboundListener.FilterChains[j].FilterChainMatch.PrefixRanges[0].PrefixLen.String() + + ":" + outboundListener.FilterChains[j].FilterChainMatch.DestinationPort.String() + } + if len(outboundListener.FilterChains[j].FilterChainMatch.ServerNames) > 0 { + sj += outboundListener.FilterChains[j].FilterChainMatch.ServerNames[0] + } + + return si < sj }) // Add a catch-all filter chain that acts as a TCP proxy to destinations outside the mesh @@ -341,11 +397,11 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. if err != nil { return nil, err } - outboundListener.FilterChains = append(outboundListener.FilterChains, filterChain) + outboundListener.DefaultFilterChain = filterChain } // Only add the outbound listener if configured. - if len(outboundListener.FilterChains) > 0 { + if len(outboundListener.FilterChains) > 0 || outboundListener.DefaultFilterChain != nil { resources = append(resources, outboundListener) } } @@ -456,6 +512,32 @@ func makeFilterChainMatchFromAddrs(addrs map[string]struct{}) *envoy_listener_v3 } } +func makeFilterChainMatchFromAddressWithPort(address string, port int) *envoy_listener_v3.FilterChainMatch { + ranges := make([]*envoy_core_v3.CidrRange, 0) + + ip := net.ParseIP(address) + if ip == nil { + return &envoy_listener_v3.FilterChainMatch{ + ServerNames: []string{address}, + DestinationPort: &wrappers.UInt32Value{Value: uint32(port)}, + } + } + + pfxLen := uint32(32) + if ip.To4() == nil { + pfxLen = 128 + } + ranges = append(ranges, &envoy_core_v3.CidrRange{ + AddressPrefix: address, + PrefixLen: &wrappers.UInt32Value{Value: pfxLen}, + }) + + return &envoy_listener_v3.FilterChainMatch{ + PrefixRanges: ranges, + DestinationPort: &wrappers.UInt32Value{Value: uint32(port)}, + } +} + func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) { var path structs.ExposePath @@ -1223,7 +1305,7 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener( } for _, svc := range cfgSnap.TerminatingGateway.ValidDestinations() { - clusterName := connect.ServiceSNI(svc.Name, "", svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) + clusterName := clusterNameForDestination(cfgSnap, svc.Name, svc.NamespaceOrDefault(), svc.PartitionOrDefault()) intentions := cfgSnap.TerminatingGateway.Intentions[svc] svcConfig := cfgSnap.TerminatingGateway.ServiceConfigs[svc] @@ -1240,11 +1322,7 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener( } var dest *structs.DestinationConfig - if cfgSnap.TerminatingGateway.DestinationServices[svc].ServiceKind == structs.GatewayServiceKindDestination { - dest = &svcConfig.Destination - } else { - return nil, fmt.Errorf("invalid gateway service for destination %s", svc.Name) - } + dest = &svcConfig.Destination clusterChain, err := s.makeFilterChainTerminatingGateway(cfgSnap, clusterName, svc, intentions, cfg.Protocol, dest) if err != nil { return nil, fmt.Errorf("failed to make filter chain for cluster %q: %v", clusterName, err) @@ -1299,19 +1377,10 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg. return nil, err } - var filterChain *envoy_listener_v3.FilterChain - if dest != nil { - filterChain = &envoy_listener_v3.FilterChain{ - FilterChainMatch: makeDestinationFilterChainMatch(cluster, dest), - Filters: make([]*envoy_listener_v3.Filter, 0, 3), - TransportSocket: transportSocket, - } - } else { - filterChain = &envoy_listener_v3.FilterChain{ - FilterChainMatch: makeSNIFilterChainMatch(cluster), - Filters: make([]*envoy_listener_v3.Filter, 0, 3), - TransportSocket: transportSocket, - } + filterChain := &envoy_listener_v3.FilterChain{ + FilterChainMatch: makeSNIFilterChainMatch(cluster), + Filters: make([]*envoy_listener_v3.Filter, 0, 3), + TransportSocket: transportSocket, } // This controls if we do L4 or L7 intention checks. @@ -1335,28 +1404,16 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg. filterChain.Filters = append(filterChain.Filters, authFilter) } - // For Destinations of Hostname types, we use the dynamic forward proxy filter since this could be - // a wildcard match. We also send to the dynamic forward cluster - if dest != nil && dest.HasHostname() { - dynamicFilter, err := makeSNIDynamicForwardProxyFilter(dest.Port) - if err != nil { - return nil, err - } - filterChain.Filters = append(filterChain.Filters, dynamicFilter) - cluster = dynamicForwardProxyClusterName - } - // Lastly we setup the actual proxying component. For L4 this is a straight // tcp proxy. For L7 this is a very hands-off HTTP proxy just to inject an // HTTP filter to do intention checks here instead. opts := listenerFilterOpts{ - protocol: protocol, - filterName: fmt.Sprintf("%s.%s.%s.%s", service.Name, service.NamespaceOrDefault(), service.PartitionOrDefault(), cfgSnap.Datacenter), - routeName: cluster, // Set cluster name for route config since each will have its own - cluster: cluster, - statPrefix: "upstream.", - routePath: "", - useDynamicForwardProxy: dest != nil && dest.HasHostname(), + protocol: protocol, + filterName: fmt.Sprintf("%s.%s.%s.%s", service.Name, service.NamespaceOrDefault(), service.PartitionOrDefault(), cfgSnap.Datacenter), + routeName: cluster, // Set cluster name for route config since each will have its own + cluster: cluster, + statPrefix: "upstream.", + routePath: "", } if useHTTPFilter { @@ -1387,6 +1444,7 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg. filter, err := makeListenerFilter(opts) if err != nil { + s.Logger.Error("failed to make listener", "cluster", cluster, "error", err) return nil, err } filterChain.Filters = append(filterChain.Filters, filter) @@ -1394,23 +1452,6 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg. return filterChain, nil } -func makeDestinationFilterChainMatch(cluster string, dest *structs.DestinationConfig) *envoy_listener_v3.FilterChainMatch { - // For hostname and wildcard destinations, we match on the address. - - // For IP Destinations, use the alias SNI name to match - ip := net.ParseIP(dest.Address) - if ip != nil { - return &envoy_listener_v3.FilterChainMatch{ - ServerNames: []string{cluster}, - } - } - - // For hostname and wildcard destinations, we match on the address in the Destination - return &envoy_listener_v3.FilterChainMatch{ - ServerNames: []string{dest.Address}, - } -} - func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, cfgSnap *proxycfg.ConfigSnapshot) (*envoy_listener_v3.Listener, error) { tlsInspector, err := makeTLSInspectorListenerFilter() if err != nil { @@ -1705,12 +1746,15 @@ func (s *ResourceGenerator) getAndModifyUpstreamConfigForListener( cfg.EnvoyListenerJSON = "" } } - protocol := cfg.Protocol - if protocol == "" { - protocol = chain.Protocol - } - if protocol == "" { + if chain != nil { + if protocol == "" { + protocol = chain.Protocol + } + if protocol == "" { + protocol = "tcp" + } + } else { protocol = "tcp" } @@ -1761,19 +1805,18 @@ func (s *ResourceGenerator) getAndModifyUpstreamConfigForPeeredListener( } type listenerFilterOpts struct { - useRDS bool - protocol string - filterName string - routeName string - cluster string - statPrefix string - routePath string - requestTimeoutMs *int - ingressGateway bool - httpAuthzFilter *envoy_http_v3.HttpFilter - forwardClientDetails bool - forwardClientPolicy envoy_http_v3.HttpConnectionManager_ForwardClientCertDetails - useDynamicForwardProxy bool + useRDS bool + protocol string + filterName string + routeName string + cluster string + statPrefix string + routePath string + requestTimeoutMs *int + ingressGateway bool + httpAuthzFilter *envoy_http_v3.HttpFilter + forwardClientDetails bool + forwardClientPolicy envoy_http_v3.HttpConnectionManager_ForwardClientCertDetails } func makeListenerFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error) { @@ -1806,13 +1849,6 @@ func makeSNIClusterFilter() (*envoy_listener_v3.Filter, error) { return makeFilter("envoy.filters.network.sni_cluster", &envoy_sni_cluster_v3.SniCluster{}) } -func makeSNIDynamicForwardProxyFilter(upstreamPort int) (*envoy_listener_v3.Filter, error) { - return makeFilter("envoy.filters.network.sni_dynamic_forward_proxy", &envoy_sni_dynamic_forward_proxy_v3.FilterConfig{ - DnsCacheConfig: getCommonDNSCacheConfiguration(), - PortSpecifier: &envoy_sni_dynamic_forward_proxy_v3.FilterConfig_PortValue{PortValue: uint32(upstreamPort)}, - }) -} - func makeTCPProxyFilter(filterName, cluster, statPrefix string) (*envoy_listener_v3.Filter, error) { cfg := &envoy_tcp_proxy_v3.TcpProxy{ StatPrefix: makeStatPrefix(statPrefix, filterName), diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index ca6750fb58..3055b44368 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -776,12 +776,6 @@ func TestListenersFromSnapshot(t *testing.T) { name: "transparent-proxy-terminating-gateway", create: proxycfg.TestConfigSnapshotTransparentProxyTerminatingGatewayCatalogDestinationsOnly, }, - { - name: "transparent-proxy-terminating-gateway-destinations-only", - create: func(t testinf.T) *proxycfg.ConfigSnapshot { - return proxycfg.TestConfigSnapshotTerminatingGatewayDestinations(t, true, nil) - }, - }, } latestEnvoyVersion := proxysupport.EnvoyVersions[0] diff --git a/agent/xds/rbac.go b/agent/xds/rbac.go index 11b37fc321..cd1424ca93 100644 --- a/agent/xds/rbac.go +++ b/agent/xds/rbac.go @@ -24,10 +24,7 @@ func makeRBACNetworkFilter( localInfo rbacLocalInfo, peerTrustBundles []*pbpeering.PeeringTrustBundle, ) (*envoy_listener_v3.Filter, error) { - rules, err := makeRBACRules(intentions, intentionDefaultAllow, localInfo, false, peerTrustBundles) - if err != nil { - return nil, err - } + rules := makeRBACRules(intentions, intentionDefaultAllow, localInfo, false, peerTrustBundles) cfg := &envoy_network_rbac_v3.RBAC{ StatPrefix: "connect_authz", @@ -42,10 +39,7 @@ func makeRBACHTTPFilter( localInfo rbacLocalInfo, peerTrustBundles []*pbpeering.PeeringTrustBundle, ) (*envoy_http_v3.HttpFilter, error) { - rules, err := makeRBACRules(intentions, intentionDefaultAllow, localInfo, true, peerTrustBundles) - if err != nil { - return nil, err - } + rules := makeRBACRules(intentions, intentionDefaultAllow, localInfo, true, peerTrustBundles) cfg := &envoy_http_rbac_v3.RBAC{ Rules: rules, @@ -485,7 +479,7 @@ func makeRBACRules( localInfo rbacLocalInfo, isHTTP bool, peerTrustBundles []*pbpeering.PeeringTrustBundle, -) (*envoy_rbac_v3.RBAC, error) { +) *envoy_rbac_v3.RBAC { // TODO(banks,rb): Implement revocation list checking? // TODO(peering): mkeeler asked that these maps come from proxycfg instead of @@ -565,7 +559,7 @@ func makeRBACRules( if len(rbac.Policies) == 0 { rbac.Policies = nil } - return rbac, nil + return rbac } func optimizePrincipals(orig []*envoy_rbac_v3.Principal) []*envoy_rbac_v3.Principal { diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 45070a8c17..983f1bb44f 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -149,6 +149,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotPeering, }, } + tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...) tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...) tests = append(tests, getEnterpriseGoldenTestCases()...) @@ -166,6 +167,21 @@ func TestAllResourcesFromSnapshot(t *testing.T) { } } +func getConnectProxyTransparentProxyGoldenTestCases() []goldenTestCase { + return []goldenTestCase{ + { + name: "transparent-proxy-destination", + create: proxycfg.TestConfigSnapshotTransparentProxyDestination, + }, + { + name: "transparent-proxy-terminating-gateway-destinations-only", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotTerminatingGatewayDestinations(t, true, nil) + }, + }, + } +} + func getMeshGatewayPeeringGoldenTestCases() []goldenTestCase { return []goldenTestCase{ { diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 6faa1fa674..dd0beacb23 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -86,47 +86,80 @@ func (s *ResourceGenerator) routesForTerminatingGateway(cfgSnap *proxycfg.Config var resources []proto.Message for _, svc := range cfgSnap.TerminatingGateway.ValidServices() { clusterName := connect.ServiceSNI(svc.Name, "", svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - resolver, hasResolver := cfgSnap.TerminatingGateway.ServiceResolvers[svc] - - svcConfig := cfgSnap.TerminatingGateway.ServiceConfigs[svc] - - cfg, err := ParseProxyConfig(svcConfig.ProxyConfig) + routes, err := s.makeRoutes(cfgSnap, svc, clusterName, true) if err != nil { - return nil, fmt.Errorf("failed to parse upstream config: %v", err) + return nil, err } - if !structs.IsProtocolHTTPLike(cfg.Protocol) { - // Routes can only be defined for HTTP services - continue - } - - if !hasResolver { - // Use a zero value resolver with no timeout and no subsets - resolver = &structs.ServiceResolverConfigEntry{} - } - - var lb *structs.LoadBalancer - if resolver.LoadBalancer != nil { - lb = resolver.LoadBalancer - } - route, err := makeNamedDefaultRouteWithLB(clusterName, lb, true) - if err != nil { - s.Logger.Error("failed to make route", "cluster", clusterName, "error", err) - continue - } - resources = append(resources, route) - - // If there is a service-resolver for this service then also setup routes for each subset - for name := range resolver.Subsets { - clusterName = connect.ServiceSNI(svc.Name, name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - route, err := makeNamedDefaultRouteWithLB(clusterName, lb, true) - if err != nil { - s.Logger.Error("failed to make route", "cluster", clusterName, "error", err) - continue - } - resources = append(resources, route) + if routes != nil { + resources = append(resources, routes...) } } + for _, svc := range cfgSnap.TerminatingGateway.ValidDestinations() { + clusterName := clusterNameForDestination(cfgSnap, svc.Name, svc.NamespaceOrDefault(), svc.PartitionOrDefault()) + routes, err := s.makeRoutes(cfgSnap, svc, clusterName, false) + if err != nil { + return nil, err + } + if routes != nil { + resources = append(resources, routes...) + } + } + + return resources, nil +} + +func (s *ResourceGenerator) makeRoutes( + cfgSnap *proxycfg.ConfigSnapshot, + svc structs.ServiceName, + clusterName string, + autoHostRewrite bool) ([]proto.Message, error) { + resolver, hasResolver := cfgSnap.TerminatingGateway.ServiceResolvers[svc] + + svcConfig := cfgSnap.TerminatingGateway.ServiceConfigs[svc] + + cfg, err := ParseProxyConfig(svcConfig.ProxyConfig) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn( + "failed to parse Proxy.Config", + "service", svc.String(), + "error", err, + ) + } + if !structs.IsProtocolHTTPLike(cfg.Protocol) { + // Routes can only be defined for HTTP services + return nil, nil + } + + if !hasResolver { + // Use a zero value resolver with no timeout and no subsets + resolver = &structs.ServiceResolverConfigEntry{} + } + + var resources []proto.Message + var lb *structs.LoadBalancer + if resolver.LoadBalancer != nil { + lb = resolver.LoadBalancer + } + route, err := makeNamedDefaultRouteWithLB(clusterName, lb, autoHostRewrite) + if err != nil { + s.Logger.Error("failed to make route", "cluster", clusterName, "error", err) + return nil, err + } + resources = append(resources, route) + + // If there is a service-resolver for this service then also setup routes for each subset + for name := range resolver.Subsets { + clusterName = connect.ServiceSNI(svc.Name, name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) + route, err := makeNamedDefaultRouteWithLB(clusterName, lb, true) + if err != nil { + s.Logger.Error("failed to make route", "cluster", clusterName, "error", err) + return nil, err + } + resources = append(resources, route) + } return resources, nil } diff --git a/agent/xds/testdata/clusters/transparent-proxy-destination.latest.golden b/agent/xds/testdata/clusters/transparent-proxy-destination.latest.golden new file mode 100644 index 0000000000..1a5311ab9d --- /dev/null +++ b/agent/xds/testdata/clusters/transparent-proxy-destination.latest.golden @@ -0,0 +1,255 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": { + + }, + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db" + } + ] + } + }, + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "destination~google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "destination~google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/google" + } + ] + } + }, + "sni": "destination~google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "destination~kafka.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "altStatName": "destination~kafka.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/kafka" + } + ] + } + }, + "sni": "destination~kafka.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": { + + }, + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target" + }, + { + "exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target" + } + ] + } + }, + "sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/transparent-proxy-terminating-gateway-destinations-only.latest.golden b/agent/xds/testdata/clusters/transparent-proxy-terminating-gateway-destinations-only.latest.golden index cd99d12dde..3cc818f59c 100644 --- a/agent/xds/testdata/clusters/transparent-proxy-terminating-gateway-destinations-only.latest.golden +++ b/agent/xds/testdata/clusters/transparent-proxy-terminating-gateway-destinations-only.latest.golden @@ -3,26 +3,39 @@ "resources": [ { "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "dynamic_forward_proxy_cluster", - "clusterType": { - "name": "envoy.clusters.dynamic_forward_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig", - "dnsCacheConfig": { - "name": "dynamic_forward_proxy_cache_config" - } - } - }, - "connectTimeout": "5s", - "lbPolicy": "CLUSTER_PROVIDED" - }, - { - "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", - "name": "external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "name": "destination~external-IP-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", "type": "STATIC", "connectTimeout": "5s", "loadAssignment": { - "clusterName": "external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "clusterName": "destination~external-IP-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "192.168.0.2", + "portValue": 80 + } + } + } + } + ] + } + ] + }, + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "destination~external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "destination~external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", "endpoints": [ { "lbEndpoints": [ @@ -42,6 +55,64 @@ }, "outlierDetection": { + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "destination~external-hostname-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "destination~external-hostname-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "httpbin.org", + "portValue": 80 + } + } + } + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "destination~external-hostname-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "destination~external-hostname-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "api.hashicorp.com", + "portValue": 8089 + } + } + } + } + ] + } + ] + }, + "dnsRefreshRate": "10s", + "outlierDetection": { + } } ], diff --git a/agent/xds/testdata/endpoints/transparent-proxy-destination.latest.golden b/agent/xds/testdata/endpoints/transparent-proxy-destination.latest.golden new file mode 100644 index 0000000000..d51ea93da4 --- /dev/null +++ b/agent/xds/testdata/endpoints/transparent-proxy-destination.latest.golden @@ -0,0 +1,119 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "destination~google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.168.0.1", + "portValue": 8443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "destination~kafka.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.168.0.1", + "portValue": 8443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.10.1.1", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "10.20.1.2", + "portValue": 8080 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/transparent-proxy-terminating-gateway-destinations-only.latest.golden b/agent/xds/testdata/endpoints/transparent-proxy-terminating-gateway-destinations-only.latest.golden new file mode 100644 index 0000000000..8504dae2b8 --- /dev/null +++ b/agent/xds/testdata/endpoints/transparent-proxy-terminating-gateway-destinations-only.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/transparent-proxy-destination.latest.golden b/agent/xds/testdata/listeners/transparent-proxy-destination.latest.golden new file mode 100644 index 0000000000..c61302c5e4 --- /dev/null +++ b/agent/xds/testdata/listeners/transparent-proxy-destination.latest.golden @@ -0,0 +1,185 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "db:127.0.0.1:9191", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 9191 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.db.default.default.dc1", + "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "outbound_listener:127.0.0.1:15001", + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 15001 + } + }, + "filterChains": [ + { + "filterChainMatch": { + "destinationPort": 9093, + "prefixRanges": [ + { + "addressPrefix": "192.168.2.1", + "prefixLen": 32 + } + ] + }, + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.destination~kafka.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "cluster": "destination~kafka.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + }, + { + "filterChainMatch": { + "destinationPort": 443, + "serverNames": [ + "www.google.com" + ] + }, + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.destination~google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "cluster": "destination~google.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "listenerFilters": [ + { + "name": "envoy.filters.listener.original_dst", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.listener.original_dst.v3.OriginalDst" + } + }, + { + "name": "envoy.filters.listener.tls_inspector", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector" + } + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "prepared_query:geo-cache:127.10.10.10:8181", + "address": { + "socketAddress": { + "address": "127.10.10.10", + "portValue": 8181 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.prepared_query_geo-cache", + "cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "public_listener:0.0.0.0:9999", + "address": { + "socketAddress": { + "address": "0.0.0.0", + "portValue": 9999 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "public_listener", + "cluster": "local_app" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + } + ], + "trafficDirection": "INBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/transparent-proxy-dial-instances-directly.latest.golden b/agent/xds/testdata/listeners/transparent-proxy-dial-instances-directly.latest.golden index d6f3ea51d8..ddfaa45d5f 100644 --- a/agent/xds/testdata/listeners/transparent-proxy-dial-instances-directly.latest.golden +++ b/agent/xds/testdata/listeners/transparent-proxy-dial-instances-directly.latest.golden @@ -99,20 +99,20 @@ } } ] - }, - { - "filters": [ - { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.original-destination", - "cluster": "original-destination" - } - } - ] } ], + "defaultFilterChain": { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.original-destination", + "cluster": "original-destination" + } + } + ] + }, "listenerFilters": [ { "name": "envoy.filters.listener.original_dst", diff --git a/agent/xds/testdata/listeners/transparent-proxy-http-upstream.latest.golden b/agent/xds/testdata/listeners/transparent-proxy-http-upstream.latest.golden index b6f00f2cdc..7c4a0a6221 100644 --- a/agent/xds/testdata/listeners/transparent-proxy-http-upstream.latest.golden +++ b/agent/xds/testdata/listeners/transparent-proxy-http-upstream.latest.golden @@ -92,20 +92,20 @@ } } ] - }, - { - "filters": [ - { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.original-destination", - "cluster": "original-destination" - } - } - ] } ], + "defaultFilterChain": { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.original-destination", + "cluster": "original-destination" + } + } + ] + }, "listenerFilters": [ { "name": "envoy.filters.listener.original_dst", diff --git a/agent/xds/testdata/listeners/transparent-proxy-terminating-gateway-destinations-only.latest.golden b/agent/xds/testdata/listeners/transparent-proxy-terminating-gateway-destinations-only.latest.golden index 80bfab3c9e..e281eb734d 100644 --- a/agent/xds/testdata/listeners/transparent-proxy-terminating-gateway-destinations-only.latest.golden +++ b/agent/xds/testdata/listeners/transparent-proxy-terminating-gateway-destinations-only.latest.golden @@ -14,36 +14,54 @@ { "filterChainMatch": { "serverNames": [ - "*.hashicorp.com" + "destination~external-IP-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { - "name": "envoy.filters.network.rbac", + "name": "envoy.filters.network.http_connection_manager", "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", - "rules": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "upstream.external-IP-HTTP.default.default.dc1", + "rds": { + "configSource": { + "ads": { + }, + "resourceApiVersion": "V3" + }, + "routeConfigName": "destination~external-IP-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, - "statPrefix": "connect_authz" - } - }, - { - "name": "envoy.filters.network.sni_dynamic_forward_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3.FilterConfig", - "dnsCacheConfig": { - "name": "dynamic_forward_proxy_cache_config" + "httpFilters": [ + { + "name": "envoy.filters.http.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "rules": { + + } + } + }, + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": { + + } }, - "portValue": 8089 - } - }, - { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.external-hostname-TCP.default.default.dc1", - "cluster": "dynamic_forward_proxy_cluster" + "forwardClientCertDetails": "APPEND_FORWARD", + "setCurrentClientCertDetails": { + "subject": true, + "cert": true, + "chain": true, + "dns": true, + "uri": true + } } } ], @@ -78,7 +96,7 @@ { "filterChainMatch": { "serverNames": [ - "external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "destination~external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ @@ -97,7 +115,143 @@ "typedConfig": { "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", "statPrefix": "upstream.external-IP-TCP.default.default.dc1", - "cluster": "external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "cluster": "destination~external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "placeholder.crt\n" + }, + "privateKey": { + "inlineString": "placeholder.key\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + }, + { + "filterChainMatch": { + "serverNames": [ + "destination~external-hostname-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "upstream.external-hostname-HTTP.default.default.dc1", + "rds": { + "configSource": { + "ads": { + + }, + "resourceApiVersion": "V3" + }, + "routeConfigName": "destination~external-hostname-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "rules": { + + } + } + }, + { + "name": "envoy.filters.http.router", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "randomSampling": { + + } + }, + "forwardClientCertDetails": "APPEND_FORWARD", + "setCurrentClientCertDetails": { + "subject": true, + "cert": true, + "chain": true, + "dns": true, + "uri": true + } + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "placeholder.crt\n" + }, + "privateKey": { + "inlineString": "placeholder.key\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": true + } + } + }, + { + "filterChainMatch": { + "serverNames": [ + "destination~external-hostname-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.filters.network.rbac", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", + "rules": { + + }, + "statPrefix": "connect_authz" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.external-hostname-TCP.default.default.dc1", + "cluster": "destination~external-hostname-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" } } ], diff --git a/agent/xds/testdata/listeners/transparent-proxy.latest.golden b/agent/xds/testdata/listeners/transparent-proxy.latest.golden index ca8b75eb2d..ee4c90857d 100644 --- a/agent/xds/testdata/listeners/transparent-proxy.latest.golden +++ b/agent/xds/testdata/listeners/transparent-proxy.latest.golden @@ -59,20 +59,20 @@ } } ] - }, - { - "filters": [ - { - "name": "envoy.filters.network.tcp_proxy", - "typedConfig": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "statPrefix": "upstream.original-destination", - "cluster": "original-destination" - } - } - ] } ], + "defaultFilterChain": { + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "statPrefix": "upstream.original-destination", + "cluster": "original-destination" + } + } + ] + }, "listenerFilters": [ { "name": "envoy.filters.listener.original_dst", diff --git a/agent/xds/testdata/routes/transparent-proxy-destination.latest.golden b/agent/xds/testdata/routes/transparent-proxy-destination.latest.golden new file mode 100644 index 0000000000..9c050cbe6b --- /dev/null +++ b/agent/xds/testdata/routes/transparent-proxy-destination.latest.golden @@ -0,0 +1,5 @@ +{ + "versionInfo": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/transparent-proxy-terminating-gateway-destinations-only.latest.golden b/agent/xds/testdata/routes/transparent-proxy-terminating-gateway-destinations-only.latest.golden new file mode 100644 index 0000000000..5eb0990108 --- /dev/null +++ b/agent/xds/testdata/routes/transparent-proxy-terminating-gateway-destinations-only.latest.golden @@ -0,0 +1,53 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "destination~external-IP-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "virtualHosts": [ + { + "name": "destination~external-IP-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "destination~external-IP-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + }, + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "destination~external-hostname-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "virtualHosts": [ + { + "name": "destination~external-hostname-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "destination~external-hostname-HTTP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file