From 6d35edc21c91e699ca21181f4988866c1493684a Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Thu, 25 May 2023 09:54:55 -0500 Subject: [PATCH] xds: generate routes directly from API gateway snapshot (#17392) * xds generation for routes api gateway * Update gateway.go * move buildHttpRoute into xds package * Update agent/consul/discoverychain/gateway.go * remove unneeded function * convert http route code to only run for http protocol to future proof code path * Update agent/consul/discoverychain/gateway.go Co-authored-by: Mike Morris * fix tests, clean up http check logic * clean up todo * Fix casing in docstring * Fix import block, adjust docstrings * update name and comment * use constant value * use constant --------- Co-authored-by: Mike Morris Co-authored-by: Nathan Coleman --- agent/consul/discoverychain/gateway.go | 14 +++- agent/xds/listeners_apigateway.go | 2 +- agent/xds/routes.go | 103 +++++++++++++++++++++++-- 3 files changed, 107 insertions(+), 12 deletions(-) diff --git a/agent/consul/discoverychain/gateway.go b/agent/consul/discoverychain/gateway.go index 054e892e02..8c13bb59a8 100644 --- a/agent/consul/discoverychain/gateway.go +++ b/agent/consul/discoverychain/gateway.go @@ -59,10 +59,10 @@ func (l *GatewayChainSynthesizer) SetHostname(hostname string) { // single hostname can be specified in multiple routes. Routing for a given // hostname must behave based on the aggregate of all rules that apply to it. func (l *GatewayChainSynthesizer) AddHTTPRoute(route structs.HTTPRouteConfigEntry) { - l.matchesByHostname = getHostMatches(l.hostname, &route, l.matchesByHostname) + l.matchesByHostname = initHostMatches(l.hostname, &route, l.matchesByHostname) } -func getHostMatches(hostname string, route *structs.HTTPRouteConfigEntry, currentMatches map[string][]hostnameMatch) map[string][]hostnameMatch { +func initHostMatches(hostname string, route *structs.HTTPRouteConfigEntry, currentMatches map[string][]hostnameMatch) map[string][]hostnameMatch { hostnames := route.FilteredHostnames(hostname) for _, host := range hostnames { matches, ok := currentMatches[host] @@ -196,16 +196,22 @@ func (l *GatewayChainSynthesizer) consolidateHTTPRoutes() []structs.HTTPRouteCon return consolidateHTTPRoutes(l.matchesByHostname, l.suffix, l.gateway) } +// ReformatHTTPRoute takes in an HTTPRoute and reformats it to match the discovery chains generated by the gateway chain synthesizer +func ReformatHTTPRoute(route *structs.HTTPRouteConfigEntry, listener *structs.APIGatewayListener, gateway *structs.APIGatewayConfigEntry) []structs.HTTPRouteConfigEntry { + matches := initHostMatches(listener.GetHostname(), route, map[string][]hostnameMatch{}) + return consolidateHTTPRoutes(matches, listener.Name, gateway) +} + // consolidateHTTPRoutes combines all rules into the shortest possible list of routes // with one route per hostname containing all rules for that hostname. -func consolidateHTTPRoutes(matchesByHostname map[string][]hostnameMatch, suffix string, gateway *structs.APIGatewayConfigEntry) []structs.HTTPRouteConfigEntry { +func consolidateHTTPRoutes(matchesByHostname map[string][]hostnameMatch, listenerName string, gateway *structs.APIGatewayConfigEntry) []structs.HTTPRouteConfigEntry { var routes []structs.HTTPRouteConfigEntry for hostname, rules := range matchesByHostname { // Create route for this hostname route := structs.HTTPRouteConfigEntry{ Kind: structs.HTTPRoute, - Name: fmt.Sprintf("%s-%s-%s", gateway.Name, suffix, hostsKey(hostname)), + Name: fmt.Sprintf("%s-%s-%s", gateway.Name, listenerName, hostsKey(hostname)), Hostnames: []string{hostname}, Rules: make([]structs.HTTPRouteRule, 0, len(rules)), Meta: gateway.Meta, diff --git a/agent/xds/listeners_apigateway.go b/agent/xds/listeners_apigateway.go index ed6cc6a9e0..d731f121dd 100644 --- a/agent/xds/listeners_apigateway.go +++ b/agent/xds/listeners_apigateway.go @@ -43,7 +43,7 @@ func (s *ResourceGenerator) makeAPIGatewayListeners(address string, cfgSnap *pro return nil, err } - if listenerKey.Protocol == "tcp" { + if listenerCfg.Protocol == structs.ListenerProtocolTCP { // Find the upstream matching this listener // We rely on the invariant of upstreams slice always having at least 1 diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 30907002e8..0eab72b259 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -19,6 +19,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" ) @@ -36,13 +37,7 @@ func (s *ResourceGenerator) routesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) case structs.ServiceKindIngressGateway: return s.routesForIngressGateway(cfgSnap) case structs.ServiceKindAPIGateway: - // TODO Find a cleaner solution, can't currently pass unexported property types - var err error - cfgSnap.IngressGateway, err = cfgSnap.APIGateway.ToIngress(cfgSnap.Datacenter) - if err != nil { - return nil, err - } - return s.routesForIngressGateway(cfgSnap) + return s.routesForAPIGateway(cfgSnap) case structs.ServiceKindTerminatingGateway: return s.routesForTerminatingGateway(cfgSnap) case structs.ServiceKindMeshGateway: @@ -430,6 +425,85 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap return result, nil } +// routesForAPIGateway returns the xDS API representation of the +// "routes" in the snapshot. +func (s *ResourceGenerator) routesForAPIGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { + var result []proto.Message + + readyUpstreamsList := getReadyUpstreams(cfgSnap) + + for _, readyUpstreams := range readyUpstreamsList { + listenerCfg := readyUpstreams.listenerCfg + // Do not create any route configuration for TCP listeners + if listenerCfg.Protocol != structs.ListenerProtocolHTTP { + continue + } + + routeRef := readyUpstreams.routeReference + listenerKey := readyUpstreams.listenerKey + + defaultRoute := &envoy_route_v3.RouteConfiguration{ + Name: listenerKey.RouteName(), + // ValidateClusters defaults to true when defined statically and false + // when done via RDS. Re-set the reasonable value of true to prevent + // null-routing traffic. + ValidateClusters: makeBoolValue(true), + } + + route, ok := cfgSnap.APIGateway.HTTPRoutes.Get(routeRef) + if !ok { + return nil, fmt.Errorf("missing route for route reference %s:%s", routeRef.Name, routeRef.Kind) + } + + // Reformat the route here since discovery chains were indexed earlier using the + // specific naming convention in discoverychain.consolidateHTTPRoutes. If we don't + // convert our route to use the same naming convention, we won't find any chains below. + reformatedRoutes := discoverychain.ReformatHTTPRoute(route, &listenerCfg, cfgSnap.APIGateway.GatewayConfig) + + for _, reformatedRoute := range reformatedRoutes { + reformatedRoute := reformatedRoute + + upstream := buildHTTPRouteUpstream(reformatedRoute, listenerCfg) + uid := proxycfg.NewUpstreamID(&upstream) + chain := cfgSnap.APIGateway.DiscoveryChain[uid] + if chain == nil { + s.Logger.Debug("Discovery chain not found for flattened route", "discovery chain ID", uid) + continue + } + + domains := generateUpstreamAPIsDomains(listenerKey, upstream, reformatedRoute.Hostnames) + + virtualHost, err := s.makeUpstreamRouteForDiscoveryChain(cfgSnap, uid, chain, domains, false) + if err != nil { + return nil, err + } + + injectHeaderManipToVirtualHostAPIGateway(&reformatedRoute, virtualHost) + + defaultRoute.VirtualHosts = append(defaultRoute.VirtualHosts, virtualHost) + } + + if len(defaultRoute.VirtualHosts) > 0 { + result = append(result, defaultRoute) + } + } + + return result, nil +} + +func buildHTTPRouteUpstream(route structs.HTTPRouteConfigEntry, listener structs.APIGatewayListener) structs.Upstream { + return structs.Upstream{ + DestinationName: route.GetName(), + DestinationNamespace: route.NamespaceOrDefault(), + DestinationPartition: route.PartitionOrDefault(), + IngressHosts: route.Hostnames, + LocalBindPort: listener.Port, + Config: map[string]interface{}{ + "protocol": string(listener.Protocol), + }, + } +} + func makeHeadersValueOptions(vals map[string]string, add bool) []*envoy_core_v3.HeaderValueOption { opts := make([]*envoy_core_v3.HeaderValueOption, 0, len(vals)) for k, v := range vals { @@ -516,6 +590,11 @@ func generateUpstreamIngressDomains(listenerKey proxycfg.IngressListenerKey, u s return domains } +func generateUpstreamAPIsDomains(listenerKey proxycfg.APIGatewayListenerKey, u structs.Upstream, hosts []string) []string { + u.IngressHosts = hosts + return generateUpstreamIngressDomains(listenerKey, u) +} + func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain( cfgSnap *proxycfg.ConfigSnapshot, uid proxycfg.UpstreamID, @@ -1019,6 +1098,16 @@ func injectHeaderManipToRoute(dest *structs.ServiceRouteDestination, r *envoy_ro return nil } +func injectHeaderManipToVirtualHostAPIGateway(dest *structs.HTTPRouteConfigEntry, vh *envoy_route_v3.VirtualHost) { + for _, rule := range dest.Rules { + for _, header := range rule.Filters.Headers { + vh.RequestHeadersToAdd = append(vh.RequestHeadersToAdd, makeHeadersValueOptions(header.Add, true)...) + vh.RequestHeadersToAdd = append(vh.RequestHeadersToAdd, makeHeadersValueOptions(header.Set, false)...) + vh.RequestHeadersToRemove = append(vh.RequestHeadersToRemove, header.Remove...) + } + } +} + func injectHeaderManipToVirtualHost(dest *structs.IngressService, vh *envoy_route_v3.VirtualHost) error { if !dest.RequestHeaders.IsZero() { vh.RequestHeadersToAdd = append(