Update xDS routes to support ingress services with different TLS config

pull/10903/head
Paul Banks 2021-08-24 15:25:22 +01:00
parent 16b3b1c737
commit 2a3d3d3c23
10 changed files with 673 additions and 37 deletions

View File

@ -177,13 +177,17 @@ func (s *ResourceGenerator) routesForIngressGateway(
continue
}
upstreamRoute := &envoy_route_v3.RouteConfiguration{
// Depending on their TLS config, upstreams are either attached to the
// default route or have their own routes. We'll add any upstreams that
// don't have custom filter chains and routes to this.
defaultRoute := &envoy_route_v3.RouteConfiguration{
Name: listenerKey.RouteName(),
// ValidateClusters defaults to true when defined statically and false
// when done via RDS. Re-set the reasonable value of true to prevent
// null-routing traffic.
ValidateClusters: makeBoolValue(true),
}
for _, u := range upstreams {
upstreamID := u.Identifier()
chain := chains[upstreamID]
@ -197,45 +201,45 @@ func (s *ResourceGenerator) routesForIngressGateway(
return nil, err
}
// See if we need to configure any special settings on this route config
if lCfg, ok := listeners[listenerKey]; ok {
if is := findIngressServiceMatchingUpstream(lCfg, u); is != nil {
// Set up any header manipulation we need
if is.RequestHeaders != nil {
virtualHost.RequestHeadersToAdd = append(
virtualHost.RequestHeadersToAdd,
makeHeadersValueOptions(is.RequestHeaders.Add, true)...,
)
virtualHost.RequestHeadersToAdd = append(
virtualHost.RequestHeadersToAdd,
makeHeadersValueOptions(is.RequestHeaders.Set, false)...,
)
virtualHost.RequestHeadersToRemove = append(
virtualHost.RequestHeadersToRemove,
is.RequestHeaders.Remove...,
)
}
if is.ResponseHeaders != nil {
virtualHost.ResponseHeadersToAdd = append(
virtualHost.ResponseHeadersToAdd,
makeHeadersValueOptions(is.ResponseHeaders.Add, true)...,
)
virtualHost.ResponseHeadersToAdd = append(
virtualHost.ResponseHeadersToAdd,
makeHeadersValueOptions(is.ResponseHeaders.Set, false)...,
)
virtualHost.ResponseHeadersToRemove = append(
virtualHost.ResponseHeadersToRemove,
is.ResponseHeaders.Remove...,
)
}
}
// Lookup listener and service config details from ingress gateway
// definition.
lCfg, ok := listeners[listenerKey]
if !ok {
return nil, fmt.Errorf("missing ingress listener config (listener on port %d)", listenerKey.Port)
}
svc := findIngressServiceMatchingUpstream(lCfg, u)
if svc == nil {
return nil, fmt.Errorf("missing service in listener config (service %q listener on port %d)",
u.DestinationID(), listenerKey.Port)
}
upstreamRoute.VirtualHosts = append(upstreamRoute.VirtualHosts, virtualHost)
if err := injectHeaderManipToVirtualHost(svc, virtualHost); err != nil {
return nil, err
}
// See if this upstream has it's own route/filter chain
svcRouteName, err := routeNameForUpstream(lCfg, *svc)
if err != nil {
return nil, err
}
// If the routeName is the same as the default one, merge the virtual host
// to the default route
if svcRouteName == defaultRoute.Name {
defaultRoute.VirtualHosts = append(defaultRoute.VirtualHosts, virtualHost)
} else {
svcRoute := &envoy_route_v3.RouteConfiguration{
Name: svcRouteName,
ValidateClusters: makeBoolValue(true),
VirtualHosts: []*envoy_route_v3.VirtualHost{virtualHost},
}
result = append(result, svcRoute)
}
}
result = append(result, upstreamRoute)
if len(defaultRoute.VirtualHosts) > 0 {
result = append(result, defaultRoute)
}
}
return result, nil
@ -262,13 +266,20 @@ func findIngressServiceMatchingUpstream(l structs.IngressListener, u structs.Ups
// wasn't checked as it didn't matter. Assume there is only one now
// though!
wantSID := u.DestinationID()
var foundSameNSWildcard *structs.IngressService
for _, s := range l.Services {
sid := structs.NewServiceID(s.Name, &s.EnterpriseMeta)
if wantSID.Matches(sid) {
return &s
}
if s.Name == structs.WildcardSpecifier &&
s.NamespaceOrDefault() == wantSID.NamespaceOrDefault() {
foundSameNSWildcard = &s
}
}
return nil
// Didn't find an exact match. Return the wildcard from same service if we
// found one.
return foundSameNSWildcard
}
func generateUpstreamIngressDomains(listenerKey proxycfg.IngressListenerKey, u structs.Upstream) []string {
@ -753,6 +764,38 @@ func injectHeaderManipToRoute(dest *structs.ServiceRouteDestination, r *envoy_ro
return nil
}
func injectHeaderManipToVirtualHost(dest *structs.IngressService, vh *envoy_route_v3.VirtualHost) error {
if !dest.RequestHeaders.IsZero() {
vh.RequestHeadersToAdd = append(
vh.RequestHeadersToAdd,
makeHeadersValueOptions(dest.RequestHeaders.Add, true)...,
)
vh.RequestHeadersToAdd = append(
vh.RequestHeadersToAdd,
makeHeadersValueOptions(dest.RequestHeaders.Set, false)...,
)
vh.RequestHeadersToRemove = append(
vh.RequestHeadersToRemove,
dest.RequestHeaders.Remove...,
)
}
if !dest.ResponseHeaders.IsZero() {
vh.ResponseHeadersToAdd = append(
vh.ResponseHeadersToAdd,
makeHeadersValueOptions(dest.ResponseHeaders.Add, true)...,
)
vh.ResponseHeadersToAdd = append(
vh.ResponseHeadersToAdd,
makeHeadersValueOptions(dest.ResponseHeaders.Set, false)...,
)
vh.ResponseHeadersToRemove = append(
vh.ResponseHeadersToRemove,
dest.ResponseHeaders.Remove...,
)
}
return nil
}
func injectHeaderManipToWeightedCluster(split *structs.ServiceSplit, c *envoy_route_v3.WeightedCluster_ClusterWeight) error {
if !split.RequestHeaders.IsZero() {
c.RequestHeadersToAdd = append(

View File

@ -155,6 +155,30 @@ func TestRoutesFromSnapshot(t *testing.T) {
},
},
}
snap.IngressGateway.Listeners = map[proxycfg.IngressListenerKey]structs.IngressListener{
{Protocol: "http", Port: 8080}: {
Port: 8080,
Services: []structs.IngressService{
{
Name: "foo",
},
{
Name: "bar",
},
},
},
{Protocol: "http", Port: 443}: {
Port: 443,
Services: []structs.IngressService{
{
Name: "baz",
},
{
Name: "qux",
},
},
},
}
// We do not add baz/qux here so that we test the chain.IsDefault() case
entries := []structs.ConfigEntry{
@ -216,6 +240,45 @@ func TestRoutesFromSnapshot(t *testing.T) {
snap.IngressGateway.Listeners[k] = l
},
},
{
name: "ingress-with-sds-listener-level",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
// Listener-level SDS means all services share the default route.
listenerSDS: true,
}),
},
{
name: "ingress-with-sds-listener-level-wildcard",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
// Listener-level SDS means all services share the default route.
listenerSDS: true,
wildcard: true,
}),
},
{
name: "ingress-with-sds-service-level",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
listenerSDS: false,
// Services should get separate routes and no default since they all
// have custom certs.
webSDS: true,
fooSDS: true,
}),
},
{
name: "ingress-with-sds-service-level-mixed-tls",
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: setupIngressWithTwoHTTPServices(t, ingressSDSOpts{
listenerSDS: false,
// Web needs a separate route as it has custom filter chain but foo
// should use default route for listener.
webSDS: true,
fooSDS: false,
}),
},
{
name: "terminating-gateway-lb-config",
create: proxycfg.TestConfigSnapshotTerminatingGateway,
@ -585,3 +648,121 @@ func TestEnvoyLBConfig_InjectToRouteAction(t *testing.T) {
})
}
}
type ingressSDSOpts struct {
listenerSDS, webSDS, fooSDS, wildcard bool
}
// setupIngressWithTwoHTTPServices can be used with
// proxycfg.TestConfigSnapshotIngressWithRouter to generate a setup func for an
// ingress listener with multiple HTTP services and varying SDS configurations
// since those affect how we generate routes.
func setupIngressWithTwoHTTPServices(t *testing.T, o ingressSDSOpts) func(snap *proxycfg.ConfigSnapshot) {
return func(snap *proxycfg.ConfigSnapshot) {
snap.IngressGateway.TLSConfig.SDS = nil
// Setup additional HTTP service on same listener with default router
snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
{Protocol: "http", Port: 9191}: {
{
DestinationName: "web",
LocalBindPort: 9191,
IngressHosts: []string{
"www.example.com",
},
},
{
DestinationName: "foo",
LocalBindPort: 9191,
IngressHosts: []string{
"foo.example.com",
},
},
},
}
il := structs.IngressListener{
Port: 9191,
Services: []structs.IngressService{
{
Name: "web",
Hosts: []string{"www.example.com"},
},
{
Name: "foo",
Hosts: []string{"foo.example.com"},
},
},
}
// Now set the appropriate SDS configs
if o.listenerSDS {
il.TLS = &structs.GatewayTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "listener-cluster",
CertResource: "listener-cert",
},
}
}
if o.webSDS {
il.Services[0].TLS = &structs.GatewayServiceTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "web-cluster",
CertResource: "www-cert",
},
}
}
if o.fooSDS {
il.Services[1].TLS = &structs.GatewayServiceTLSConfig{
SDS: &structs.GatewayTLSSDSConfig{
ClusterName: "foo-cluster",
CertResource: "foo-cert",
},
}
}
if o.wildcard {
// undo all that and set just a single wildcard config with no TLS to test
// the lookup path where we have to compare an actual resolved upstream to
// a wildcard config.
il.Services = []structs.IngressService{
{
Name: "*",
},
}
// We also don't support user-specified hosts with wildcard so remove
// those from the upstreams.
ups := snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}]
for i := range ups {
ups[i].IngressHosts = nil
}
snap.IngressGateway.Upstreams[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = ups
}
snap.IngressGateway.Listeners[proxycfg.IngressListenerKey{Protocol: "http", Port: 9191}] = il
entries := []structs.ConfigEntry{
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "web",
ConnectTimeout: 22 * time.Second,
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "foo",
ConnectTimeout: 22 * time.Second,
},
}
webChain := discoverychain.TestCompileConfigEntries(t, "web", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
snap.IngressGateway.DiscoveryChain["web"] = webChain
snap.IngressGateway.DiscoveryChain["foo"] = fooChain
}
}

View File

@ -0,0 +1,48 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "9191",
"virtualHosts": [
{
"name": "web",
"domains": [
"web.ingress.*",
"web.ingress.*:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
},
{
"name": "foo",
"domains": [
"foo.ingress.*",
"foo.ingress.*:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,48 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "9191",
"virtualHosts": [
{
"name": "web",
"domains": [
"web.ingress.*",
"web.ingress.*:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
},
{
"name": "foo",
"domains": [
"foo.ingress.*",
"foo.ingress.*:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,48 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "9191",
"virtualHosts": [
{
"name": "web",
"domains": [
"www.example.com",
"www.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
},
{
"name": "foo",
"domains": [
"foo.example.com",
"foo.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,48 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "9191",
"virtualHosts": [
{
"name": "web",
"domains": [
"www.example.com",
"www.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
},
{
"name": "foo",
"domains": [
"foo.example.com",
"foo.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,55 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "9191",
"virtualHosts": [
{
"name": "foo",
"domains": [
"foo.example.com",
"foo.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
},
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "9191_web",
"virtualHosts": [
{
"name": "web",
"domains": [
"www.example.com",
"www.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,55 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "9191",
"virtualHosts": [
{
"name": "foo",
"domains": [
"foo.example.com",
"foo.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
},
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "9191_web",
"virtualHosts": [
{
"name": "web",
"domains": [
"www.example.com",
"www.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,55 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "9191_foo",
"virtualHosts": [
{
"name": "foo",
"domains": [
"foo.example.com",
"foo.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
},
{
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "9191_web",
"virtualHosts": [
{
"name": "web",
"domains": [
"www.example.com",
"www.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

View File

@ -0,0 +1,55 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "9191_foo",
"virtualHosts": [
{
"name": "foo",
"domains": [
"foo.example.com",
"foo.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
},
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "9191_web",
"virtualHosts": [
{
"name": "web",
"domains": [
"www.example.com",
"www.example.com:9191"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"
}