diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 1a5d6403b7..bf4df8b05d 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -177,13 +177,17 @@ func (s *ResourceGenerator) routesForIngressGateway( continue } - upstreamRoute := &envoy_route_v3.RouteConfiguration{ + // Depending on their TLS config, upstreams are either attached to the + // default route or have their own routes. We'll add any upstreams that + // don't have custom filter chains and routes to this. + defaultRoute := &envoy_route_v3.RouteConfiguration{ Name: listenerKey.RouteName(), // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. ValidateClusters: makeBoolValue(true), } + for _, u := range upstreams { upstreamID := u.Identifier() chain := chains[upstreamID] @@ -197,45 +201,45 @@ func (s *ResourceGenerator) routesForIngressGateway( return nil, err } - // See if we need to configure any special settings on this route config - if lCfg, ok := listeners[listenerKey]; ok { - if is := findIngressServiceMatchingUpstream(lCfg, u); is != nil { - // Set up any header manipulation we need - if is.RequestHeaders != nil { - virtualHost.RequestHeadersToAdd = append( - virtualHost.RequestHeadersToAdd, - makeHeadersValueOptions(is.RequestHeaders.Add, true)..., - ) - virtualHost.RequestHeadersToAdd = append( - virtualHost.RequestHeadersToAdd, - makeHeadersValueOptions(is.RequestHeaders.Set, false)..., - ) - virtualHost.RequestHeadersToRemove = append( - virtualHost.RequestHeadersToRemove, - is.RequestHeaders.Remove..., - ) - } - if is.ResponseHeaders != nil { - virtualHost.ResponseHeadersToAdd = append( - virtualHost.ResponseHeadersToAdd, - makeHeadersValueOptions(is.ResponseHeaders.Add, true)..., - ) - virtualHost.ResponseHeadersToAdd = append( - virtualHost.ResponseHeadersToAdd, - makeHeadersValueOptions(is.ResponseHeaders.Set, false)..., - ) - virtualHost.ResponseHeadersToRemove = append( - virtualHost.ResponseHeadersToRemove, - is.ResponseHeaders.Remove..., - ) - } - } + // Lookup listener and service config details from ingress gateway + // definition. + lCfg, ok := listeners[listenerKey] + if !ok { + return nil, fmt.Errorf("missing ingress listener config (listener on port %d)", listenerKey.Port) + } + svc := findIngressServiceMatchingUpstream(lCfg, u) + if svc == nil { + return nil, fmt.Errorf("missing service in listener config (service %q listener on port %d)", + u.DestinationID(), listenerKey.Port) } - upstreamRoute.VirtualHosts = append(upstreamRoute.VirtualHosts, virtualHost) + if err := injectHeaderManipToVirtualHost(svc, virtualHost); err != nil { + return nil, err + } + + // See if this upstream has it's own route/filter chain + svcRouteName, err := routeNameForUpstream(lCfg, *svc) + if err != nil { + return nil, err + } + + // If the routeName is the same as the default one, merge the virtual host + // to the default route + if svcRouteName == defaultRoute.Name { + defaultRoute.VirtualHosts = append(defaultRoute.VirtualHosts, virtualHost) + } else { + svcRoute := &envoy_route_v3.RouteConfiguration{ + Name: svcRouteName, + ValidateClusters: makeBoolValue(true), + VirtualHosts: []*envoy_route_v3.VirtualHost{virtualHost}, + } + result = append(result, svcRoute) + } } - result = append(result, upstreamRoute) + if len(defaultRoute.VirtualHosts) > 0 { + result = append(result, defaultRoute) + } } return result, nil @@ -262,13 +266,20 @@ func findIngressServiceMatchingUpstream(l structs.IngressListener, u structs.Ups // wasn't checked as it didn't matter. Assume there is only one now // though! wantSID := u.DestinationID() + var foundSameNSWildcard *structs.IngressService for _, s := range l.Services { sid := structs.NewServiceID(s.Name, &s.EnterpriseMeta) if wantSID.Matches(sid) { return &s } + if s.Name == structs.WildcardSpecifier && + s.NamespaceOrDefault() == wantSID.NamespaceOrDefault() { + foundSameNSWildcard = &s + } } - return nil + // Didn't find an exact match. Return the wildcard from same service if we + // found one. + return foundSameNSWildcard } func generateUpstreamIngressDomains(listenerKey proxycfg.IngressListenerKey, u structs.Upstream) []string { @@ -753,6 +764,38 @@ func injectHeaderManipToRoute(dest *structs.ServiceRouteDestination, r *envoy_ro return nil } +func injectHeaderManipToVirtualHost(dest *structs.IngressService, vh *envoy_route_v3.VirtualHost) error { + if !dest.RequestHeaders.IsZero() { + vh.RequestHeadersToAdd = append( + vh.RequestHeadersToAdd, + makeHeadersValueOptions(dest.RequestHeaders.Add, true)..., + ) + vh.RequestHeadersToAdd = append( + vh.RequestHeadersToAdd, + makeHeadersValueOptions(dest.RequestHeaders.Set, false)..., + ) + vh.RequestHeadersToRemove = append( + vh.RequestHeadersToRemove, + dest.RequestHeaders.Remove..., + ) + } + if !dest.ResponseHeaders.IsZero() { + vh.ResponseHeadersToAdd = append( + vh.ResponseHeadersToAdd, + makeHeadersValueOptions(dest.ResponseHeaders.Add, true)..., + ) + vh.ResponseHeadersToAdd = append( + vh.ResponseHeadersToAdd, + makeHeadersValueOptions(dest.ResponseHeaders.Set, false)..., + ) + vh.ResponseHeadersToRemove = append( + vh.ResponseHeadersToRemove, + dest.ResponseHeaders.Remove..., + ) + } + return nil +} + func injectHeaderManipToWeightedCluster(split *structs.ServiceSplit, c *envoy_route_v3.WeightedCluster_ClusterWeight) error { if !split.RequestHeaders.IsZero() { c.RequestHeadersToAdd = append( diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index 63f78b7c65..2dd5eb5a21 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -155,6 +155,30 @@ func TestRoutesFromSnapshot(t *testing.T) { }, }, } + snap.IngressGateway.Listeners = map[proxycfg.IngressListenerKey]structs.IngressListener{ + {Protocol: "http", Port: 8080}: { + Port: 8080, + Services: []structs.IngressService{ + { + Name: "foo", + }, + { + Name: "bar", + }, + }, + }, + {Protocol: "http", Port: 443}: { + Port: 443, + Services: []structs.IngressService{ + { + Name: "baz", + }, + { + Name: "qux", + }, + }, + }, + } // We do not add baz/qux here so that we test the chain.IsDefault() case entries := []structs.ConfigEntry{ @@ -216,6 +240,45 @@ func TestRoutesFromSnapshot(t *testing.T) { snap.IngressGateway.Listeners[k] = l }, }, + { + name: "ingress-with-sds-listener-level", + create: proxycfg.TestConfigSnapshotIngressWithRouter, + setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{ + // Listener-level SDS means all services share the default route. + listenerSDS: true, + }), + }, + { + name: "ingress-with-sds-listener-level-wildcard", + create: proxycfg.TestConfigSnapshotIngressWithRouter, + setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{ + // Listener-level SDS means all services share the default route. + listenerSDS: true, + wildcard: true, + }), + }, + { + name: "ingress-with-sds-service-level", + create: proxycfg.TestConfigSnapshotIngressWithRouter, + setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{ + listenerSDS: false, + // Services should get separate routes and no default since they all + // have custom certs. + webSDS: true, + fooSDS: true, + }), + }, + { + name: "ingress-with-sds-service-level-mixed-tls", + create: proxycfg.TestConfigSnapshotIngressWithRouter, + setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{ + listenerSDS: false, + // Web needs a separate route as it has custom filter chain but foo + // should use default route for listener. + webSDS: true, + fooSDS: false, + }), + }, { name: "terminating-gateway-lb-config", create: proxycfg.TestConfigSnapshotTerminatingGateway, @@ -585,3 +648,121 @@ func TestEnvoyLBConfig_InjectToRouteAction(t *testing.T) { }) } } + +type ingressSDSOpts struct { + listenerSDS, webSDS, fooSDS, wildcard bool +} + +// setupIngressWithTwoHTTPServices can be used with +// proxycfg.TestConfigSnapshotIngressWithRouter to generate a setup func for an +// ingress listener with multiple HTTP services and varying SDS configurations +// since those affect how we generate routes. +func setupIngressWithTwoHTTPServices(t *testing.T, o ingressSDSOpts) func(snap *proxycfg.ConfigSnapshot) { + return func(snap *proxycfg.ConfigSnapshot) { + + snap.IngressGateway.TLSConfig.SDS = nil + + // Setup additional HTTP service on same listener with default router + snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{ + {Protocol: "http", Port: 9191}: { + { + DestinationName: "web", + LocalBindPort: 9191, + IngressHosts: []string{ + "www.example.com", + }, + }, + { + DestinationName: "foo", + LocalBindPort: 9191, + IngressHosts: []string{ + "foo.example.com", + }, + }, + }, + } + il := structs.IngressListener{ + Port: 9191, + Services: []structs.IngressService{ + { + Name: "web", + Hosts: []string{"www.example.com"}, + }, + { + Name: "foo", + Hosts: []string{"foo.example.com"}, + }, + }, + } + + // Now set the appropriate SDS configs + if o.listenerSDS { + il.TLS = &structs.GatewayTLSConfig{ + SDS: &structs.GatewayTLSSDSConfig{ + ClusterName: "listener-cluster", + CertResource: "listener-cert", + }, + } + } + if o.webSDS { + il.Services[0].TLS = &structs.GatewayServiceTLSConfig{ + SDS: &structs.GatewayTLSSDSConfig{ + ClusterName: "web-cluster", + CertResource: "www-cert", + }, + } + } + if o.fooSDS { + il.Services[1].TLS = &structs.GatewayServiceTLSConfig{ + SDS: &structs.GatewayTLSSDSConfig{ + ClusterName: "foo-cluster", + CertResource: "foo-cert", + }, + } + } + + if o.wildcard { + // undo all that and set just a single wildcard config with no TLS to test + // the lookup path where we have to compare an actual resolved upstream to + // a wildcard config. + il.Services = []structs.IngressService{ + { + Name: "*", + }, + } + // We also don't support user-specified hosts with wildcard so remove + // those from the upstreams. + ups := snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] + for i := range ups { + ups[i].IngressHosts = nil + } + snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = ups + } + + snap.IngressGateway.Listeners[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = il + + entries := []structs.ConfigEntry{ + &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + "protocol": "http", + }, + }, + &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "web", + ConnectTimeout: 22 * time.Second, + }, + &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "foo", + ConnectTimeout: 22 * time.Second, + }, + } + webChain := discoverychain.TestCompileConfigEntries(t, "web", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...) + fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...) + snap.IngressGateway.DiscoveryChain["web"] = webChain + snap.IngressGateway.DiscoveryChain["foo"] = fooChain + } +} diff --git a/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.envoy-1-18-x.golden b/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.envoy-1-18-x.golden new file mode 100644 index 0000000000..3c251942ec --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.envoy-1-18-x.golden @@ -0,0 +1,48 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "web.ingress.*", + "web.ingress.*:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + }, + { + "name": "foo", + "domains": [ + "foo.ingress.*", + "foo.ingress.*:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.v2compat.envoy-1-16-x.golden new file mode 100644 index 0000000000..d1d8bae6c1 --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-listener-level-wildcard.v2compat.envoy-1-16-x.golden @@ -0,0 +1,48 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "9191", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "web.ingress.*", + "web.ingress.*:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + }, + { + "name": "foo", + "domains": [ + "foo.ingress.*", + "foo.ingress.*:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-listener-level.envoy-1-18-x.golden b/agent/xds/testdata/routes/ingress-with-sds-listener-level.envoy-1-18-x.golden new file mode 100644 index 0000000000..0dc02c5056 --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-listener-level.envoy-1-18-x.golden @@ -0,0 +1,48 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "www.example.com", + "www.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + }, + { + "name": "foo", + "domains": [ + "foo.example.com", + "foo.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-listener-level.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/routes/ingress-with-sds-listener-level.v2compat.envoy-1-16-x.golden new file mode 100644 index 0000000000..7261889dae --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-listener-level.v2compat.envoy-1-16-x.golden @@ -0,0 +1,48 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "9191", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "www.example.com", + "www.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + }, + { + "name": "foo", + "domains": [ + "foo.example.com", + "foo.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.envoy-1-18-x.golden b/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.envoy-1-18-x.golden new file mode 100644 index 0000000000..44ab48e588 --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.envoy-1-18-x.golden @@ -0,0 +1,55 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191", + "virtualHosts": [ + { + "name": "foo", + "domains": [ + "foo.example.com", + "foo.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + }, + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191_web", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "www.example.com", + "www.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.v2compat.envoy-1-16-x.golden new file mode 100644 index 0000000000..07fbfd855f --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-service-level-mixed-tls.v2compat.envoy-1-16-x.golden @@ -0,0 +1,55 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "9191", + "virtualHosts": [ + { + "name": "foo", + "domains": [ + "foo.example.com", + "foo.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + }, + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "9191_web", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "www.example.com", + "www.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-service-level.envoy-1-18-x.golden b/agent/xds/testdata/routes/ingress-with-sds-service-level.envoy-1-18-x.golden new file mode 100644 index 0000000000..1e207d6efc --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-service-level.envoy-1-18-x.golden @@ -0,0 +1,55 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191_foo", + "virtualHosts": [ + { + "name": "foo", + "domains": [ + "foo.example.com", + "foo.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + }, + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "9191_web", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "www.example.com", + "www.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-with-sds-service-level.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/routes/ingress-with-sds-service-level.v2compat.envoy-1-16-x.golden new file mode 100644 index 0000000000..b53d5be47c --- /dev/null +++ b/agent/xds/testdata/routes/ingress-with-sds-service-level.v2compat.envoy-1-16-x.golden @@ -0,0 +1,55 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "9191_foo", + "virtualHosts": [ + { + "name": "foo", + "domains": [ + "foo.example.com", + "foo.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + }, + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "9191_web", + "virtualHosts": [ + { + "name": "web", + "domains": [ + "www.example.com", + "www.example.com:9191" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file