From 0379dabe4e1ce51c5628e1c6940dd754d1a6cca8 Mon Sep 17 00:00:00 2001 From: Derek Menteer <105233703+hashi-derek@users.noreply.github.com> Date: Thu, 19 Jan 2023 10:55:27 -0600 Subject: [PATCH] Fix configuration merging for implicit tproxy upstreams. (#16000) (#16007) Fix configuration merging for implicit tproxy upstreams. Change the merging logic so that the wildcard upstream has correct proxy-defaults and service-defaults values combined into it. It did not previously merge all fields, and the wildcard upstream did not exist unless service-defaults existed (it ignored proxy-defaults, essentially). Change the way we fetch upstream configuration in the xDS layer so that it falls back to the wildcard when no matching upstream is found. This is what allows implicit peer upstreams to have the correct "merged" config. Change proxycfg to always watch local mesh gateway endpoints whenever a peer upstream is found. This simplifies the logic so that we do not have to inspect the "merged" configuration on peer upstreams to extract the mesh gateway mode. --- .changelog/16000.txt | 3 ++ agent/configentry/resolve.go | 62 ++++++++++++++++---------- agent/configentry/resolve_test.go | 65 ++++++++++++++++++++++++++++ agent/consul/config_endpoint_test.go | 18 ++++++++ agent/proxycfg/connect_proxy.go | 20 +++------ agent/proxycfg/naming.go | 8 ++++ agent/proxycfg/snapshot.go | 14 ++++++ agent/service_manager_test.go | 49 ++++++++++++++++++--- agent/xds/clusters.go | 12 +---- agent/xds/endpoints.go | 18 +++----- agent/xds/listeners.go | 21 +++------ 11 files changed, 211 insertions(+), 79 deletions(-) create mode 100644 .changelog/16000.txt diff --git a/.changelog/16000.txt b/.changelog/16000.txt new file mode 100644 index 0000000000..aeea16f5e7 --- /dev/null +++ b/.changelog/16000.txt @@ -0,0 +1,3 @@ +```release-note:breaking-change +connect: Fix configuration merging for transparent proxy upstreams. Proxy-defaults and service-defaults config entries were not correctly merged for implicit upstreams in transparent proxy mode and would result in some configuration not being applied. To avoid issues when upgrading, ensure that any proxy-defaults or service-defaults have correct configuration for upstreams, since all fields will now be properly used to configure proxies. +``` diff --git a/agent/configentry/resolve.go b/agent/configentry/resolve.go index 4e2fc6ad20..d88e00ea89 100644 --- a/agent/configentry/resolve.go +++ b/agent/configentry/resolve.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/hashicorp/go-hclog" + "github.com/imdario/mergo" "github.com/mitchellh/copystructure" "github.com/hashicorp/consul/agent/structs" @@ -20,10 +21,18 @@ func ComputeResolvedServiceConfig( thisReply.MeshGateway.Mode = structs.MeshGatewayModeDefault + // Store the upstream defaults under a wildcard key so that they can be applied to + // upstreams that are inferred from intentions and do not have explicit upstream configuration. + wildcard := structs.NewServiceID(structs.WildcardSpecifier, args.WithWildcardNamespace()) + wildcardUpstreamDefaults := make(map[string]interface{}) + // resolvedConfigs stores the opaque config map for each upstream and is keyed on the upstream's ID. + resolvedConfigs := make(map[structs.ServiceID]map[string]interface{}) + // TODO(freddy) Refactor this into smaller set of state store functions // Pass the WatchSet to both the service and proxy config lookups. If either is updated during the // blocking query, this function will be rerun and these state store lookups will both be current. // We use the default enterprise meta to look up the global proxy defaults because they are not namespaced. + var proxyConfGlobalProtocol string proxyConf := entries.GetProxyDefaults(args.PartitionOrDefault()) if proxyConf != nil { @@ -32,20 +41,27 @@ func ComputeResolvedServiceConfig( if err != nil { return nil, fmt.Errorf("failed to copy global proxy-defaults: %v", err) } + thisReply.ProxyConfig = mapCopy.(map[string]interface{}) thisReply.Mode = proxyConf.Mode thisReply.TransparentProxy = proxyConf.TransparentProxy thisReply.MeshGateway = proxyConf.MeshGateway thisReply.Expose = proxyConf.Expose - // Extract the global protocol from proxyConf for upstream configs. - rawProtocol := proxyConf.Config["protocol"] - if rawProtocol != nil { - var ok bool - proxyConfGlobalProtocol, ok = rawProtocol.(string) - if !ok { - return nil, fmt.Errorf("invalid protocol type %T", rawProtocol) - } + parsed, err := structs.ParseUpstreamConfigNoDefaults(mapCopy.(map[string]interface{})) + if err != nil { + return nil, fmt.Errorf("failed to parse upstream config map for proxy-defaults: %v", err) + } + proxyConfGlobalProtocol = parsed.Protocol + + // MeshGateway is strange. It's marshaled into UpstreamConfigs via the arbitrary map, but it + // uses concrete fields everywhere else. We always take the explicit definition here for + // wildcard upstreams and discard the user setting it via arbitrary map in proxy-defaults. + if err := mergo.MergeWithOverwrite(&wildcardUpstreamDefaults, mapCopy); err != nil { + return nil, fmt.Errorf("failed to merge upstream config map for proxy-defaults: %v", err) + } + if !proxyConf.MeshGateway.IsZero() { + wildcardUpstreamDefaults["mesh_gateway"] = proxyConf.MeshGateway } } @@ -62,6 +78,7 @@ func ComputeResolvedServiceConfig( } if serviceConf.MeshGateway.Mode != structs.MeshGatewayModeDefault { thisReply.MeshGateway.Mode = serviceConf.MeshGateway.Mode + wildcardUpstreamDefaults["mesh_gateway"] = serviceConf.MeshGateway } if serviceConf.TransparentProxy.OutboundListenerPort != 0 { thisReply.TransparentProxy.OutboundListenerPort = serviceConf.TransparentProxy.OutboundListenerPort @@ -134,10 +151,8 @@ func ComputeResolvedServiceConfig( // Then store upstreams inferred from service-defaults and mapify the overrides. var ( - upstreamOverrides = make(map[structs.ServiceID]*structs.UpstreamConfig) upstreamDefaults *structs.UpstreamConfig - // resolvedConfigs stores the opaque config map for each upstream and is keyed on the upstream's ID. - resolvedConfigs = make(map[structs.ServiceID]map[string]interface{}) + upstreamOverrides = make(map[structs.ServiceID]*structs.UpstreamConfig) ) if serviceConf != nil && serviceConf.UpstreamConfig != nil { for i, override := range serviceConf.UpstreamConfig.Overrides { @@ -156,23 +171,24 @@ func ComputeResolvedServiceConfig( } if serviceConf.UpstreamConfig.Defaults != nil { upstreamDefaults = serviceConf.UpstreamConfig.Defaults - if upstreamDefaults.MeshGateway.Mode == structs.MeshGatewayModeDefault { upstreamDefaults.MeshGateway.Mode = thisReply.MeshGateway.Mode } + upstreamDefaults.MergeInto(wildcardUpstreamDefaults) + // Always add the wildcard upstream if a service-defaults default-upstream was configured. + resolvedConfigs[wildcard] = wildcardUpstreamDefaults + } + } - // Store the upstream defaults under a wildcard key so that they can be applied to - // upstreams that are inferred from intentions and do not have explicit upstream configuration. - cfgMap := make(map[string]interface{}) - upstreamDefaults.MergeInto(cfgMap) - - if !args.MeshGateway.IsZero() { - cfgMap["mesh_gateway"] = args.MeshGateway - } + if !args.MeshGateway.IsZero() { + wildcardUpstreamDefaults["mesh_gateway"] = args.MeshGateway + } - wildcard := structs.NewServiceID(structs.WildcardSpecifier, args.WithWildcardNamespace()) - resolvedConfigs[wildcard] = cfgMap - } + // Add the wildcard upstream if any fields were populated (it may have been already + // added if a service-defaults exists). We likely could always add it without issues, + // but this has been existing behavior, and many unit tests would break. + if len(wildcardUpstreamDefaults) > 0 { + resolvedConfigs[wildcard] = wildcardUpstreamDefaults } for upstream := range seenUpstreams { diff --git a/agent/configentry/resolve_test.go b/agent/configentry/resolve_test.go index 3039d09b0b..adbcbe73dc 100644 --- a/agent/configentry/resolve_test.go +++ b/agent/configentry/resolve_test.go @@ -120,6 +120,14 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) { want: &structs.ServiceConfigResponse{ MeshGateway: remoteMeshGW, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ + { + Upstream: wildcard, + Config: map[string]interface{}{ + "mesh_gateway": structs.MeshGatewayConfig{ + Mode: structs.MeshGatewayModeRemote, + }, + }, + }, { Upstream: uid, Config: map[string]interface{}{ @@ -129,6 +137,57 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) { }, }, }, + { + name: "proxy inherits kitchen sink from proxy-defaults", + args: args{ + scReq: &structs.ServiceConfigRequest{ + Name: "sid", + }, + entries: &ResolvedServiceConfigSet{ + ProxyDefaults: map[string]*structs.ProxyConfigEntry{ + acl.DefaultEnterpriseMeta().PartitionOrDefault(): { + Config: map[string]interface{}{ + "foo": "bar", + }, + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.ExposePath{}, + }, + Mode: structs.ProxyModeTransparent, + MeshGateway: remoteMeshGW, + TransparentProxy: structs.TransparentProxyConfig{ + OutboundListenerPort: 6666, + DialedDirectly: true, + }, + }, + }, + }, + }, + want: &structs.ServiceConfigResponse{ + ProxyConfig: map[string]interface{}{ + "foo": "bar", + }, + Expose: structs.ExposeConfig{ + Checks: true, + Paths: []structs.ExposePath{}, + }, + Mode: structs.ProxyModeTransparent, + MeshGateway: remoteMeshGW, + TransparentProxy: structs.TransparentProxyConfig{ + OutboundListenerPort: 6666, + DialedDirectly: true, + }, + UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ + { + Upstream: wildcard, + Config: map[string]interface{}{ + "foo": "bar", + "mesh_gateway": remoteMeshGW, + }, + }, + }, + }, + }, { name: "proxy upstream mesh-gateway inherits service-defaults", args: args{ @@ -153,6 +212,12 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) { want: &structs.ServiceConfigResponse{ MeshGateway: noneMeshGW, // service-defaults has a higher precedence. UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ + { + Upstream: wildcard, + Config: map[string]interface{}{ + "mesh_gateway": noneMeshGW, + }, + }, { Upstream: uid, Config: map[string]interface{}{ diff --git a/agent/consul/config_endpoint_test.go b/agent/consul/config_endpoint_test.go index fa06cf9b9f..34e9ce72b3 100644 --- a/agent/consul/config_endpoint_test.go +++ b/agent/consul/config_endpoint_test.go @@ -1057,6 +1057,9 @@ func TestConfigEntry_ResolveServiceConfig(t *testing.T) { "protocol": "http", }, UpstreamConfigs: map[string]map[string]interface{}{ + "*": { + "foo": int64(1), + }, "bar": { "protocol": "grpc", }, @@ -1270,6 +1273,9 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { "protocol": "grpc", }, UpstreamConfigs: map[string]map[string]interface{}{ + "*": { + "protocol": "grpc", + }, "mysql": { "protocol": "http", }, @@ -1314,6 +1320,12 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { "protocol": "grpc", }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ + { + Upstream: wildcard, + Config: map[string]interface{}{ + "protocol": "grpc", + }, + }, { Upstream: cache, Config: map[string]interface{}{ @@ -2052,6 +2064,9 @@ func TestConfigEntry_ResolveServiceConfig_UpstreamProxyDefaultsProtocol(t *testi "protocol": "http", }, UpstreamConfigs: map[string]map[string]interface{}{ + "*": { + "protocol": "http", + }, "bar": { "protocol": "http", }, @@ -2107,6 +2122,9 @@ func TestConfigEntry_ResolveServiceConfig_ProxyDefaultsProtocol_UsedForAllUpstre "protocol": "http", }, UpstreamConfigs: map[string]map[string]interface{}{ + "*": { + "protocol": "http", + }, "bar": { "protocol": "http", }, diff --git a/agent/proxycfg/connect_proxy.go b/agent/proxycfg/connect_proxy.go index f544ef7014..81b41db694 100644 --- a/agent/proxycfg/connect_proxy.go +++ b/agent/proxycfg/connect_proxy.go @@ -197,7 +197,7 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e case "": if u.DestinationPeer != "" { - err := s.setupWatchesForPeeredUpstream(ctx, snap.ConnectProxy, NewUpstreamID(&u), u.MeshGateway.Mode, dc) + err := s.setupWatchesForPeeredUpstream(ctx, snap.ConnectProxy, NewUpstreamID(&u), dc) if err != nil { return snap, fmt.Errorf("failed to setup watches for peered upstream %q: %w", uid.String(), err) } @@ -231,7 +231,6 @@ func (s *handlerConnectProxy) setupWatchesForPeeredUpstream( ctx context.Context, snapConnectProxy configSnapshotConnectProxy, uid UpstreamID, - mgwMode structs.MeshGatewayMode, dc string, ) error { s.logger.Trace("initializing watch of peered upstream", "upstream", uid) @@ -272,14 +271,10 @@ func (s *handlerConnectProxy) setupWatchesForPeeredUpstream( snapConnectProxy.UpstreamPeerTrustBundles.InitWatch(uid.Peer, cancel) } - // If a peered upstream is set to local mesh gw mode, - // set up a watch for them. - if mgwMode == structs.MeshGatewayModeLocal { - up := &handlerUpstreams{handlerState: s.handlerState} - up.setupWatchForLocalGWEndpoints(ctx, &snapConnectProxy.ConfigSnapshotUpstreams) - } else if mgwMode == structs.MeshGatewayModeNone { - s.logger.Warn(fmt.Sprintf("invalid mesh gateway mode 'none', defaulting to 'remote' for %q", uid)) - } + // Always watch local GW endpoints for peer upstreams so that we don't have to worry about + // the timing on whether the wildcard upstream config was fetched yet. + up := &handlerUpstreams{handlerState: s.handlerState} + up.setupWatchForLocalGWEndpoints(ctx, &snapConnectProxy.ConfigSnapshotUpstreams) return nil } @@ -329,7 +324,7 @@ func (s *handlerConnectProxy) handleUpdate(ctx context.Context, u UpdateEvent, s } seenUpstreams[uid] = struct{}{} - err := s.setupWatchesForPeeredUpstream(ctx, snap.ConnectProxy, uid, s.proxyCfg.MeshGateway.Mode, s.source.Datacenter) + err := s.setupWatchesForPeeredUpstream(ctx, snap.ConnectProxy, uid, s.source.Datacenter) if err != nil { return fmt.Errorf("failed to setup watches for peered upstream %q: %w", uid.String(), err) } @@ -402,8 +397,7 @@ func (s *handlerConnectProxy) handleUpdate(ctx context.Context, u UpdateEvent, s // Use the centralized upstream defaults if they exist and there isn't specific configuration for this upstream // This is only relevant to upstreams from intentions because for explicit upstreams the defaulting is handled // by the ResolveServiceConfig endpoint. - wildcardSID := structs.NewServiceID(structs.WildcardSpecifier, s.proxyID.WithWildcardNamespace()) - wildcardUID := NewUpstreamIDFromServiceID(wildcardSID) + wildcardUID := NewWildcardUID(&s.proxyID.EnterpriseMeta) defaults, ok := snap.ConnectProxy.UpstreamConfig[wildcardUID] if ok { u = defaults diff --git a/agent/proxycfg/naming.go b/agent/proxycfg/naming.go index 08ff216edf..0ab355160d 100644 --- a/agent/proxycfg/naming.go +++ b/agent/proxycfg/naming.go @@ -180,3 +180,11 @@ func UpstreamsToMap(us structs.Upstreams) map[UpstreamID]*structs.Upstream { } return upstreamMap } + +func NewWildcardUID(entMeta *acl.EnterpriseMeta) UpstreamID { + wildcardSID := structs.NewServiceID( + structs.WildcardSpecifier, + entMeta.WithWildcardNamespace(), + ) + return NewUpstreamIDFromServiceID(wildcardSID) +} diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 57be8bf7fe..15d9aef705 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -189,6 +189,20 @@ func (c *configSnapshotConnectProxy) IsImplicitUpstream(uid UpstreamID) bool { return intentionImplicit || peeringImplicit } +func (c *configSnapshotConnectProxy) GetUpstream(uid UpstreamID, entMeta *acl.EnterpriseMeta) (*structs.Upstream, bool) { + upstream, found := c.UpstreamConfig[uid] + // We should fallback to the wildcard defaults generated from service-defaults + proxy-defaults + // whenever we don't find the upstream config. + if !found { + wildcardUID := NewWildcardUID(entMeta) + upstream = c.UpstreamConfig[wildcardUID] + } + + explicit := upstream != nil && upstream.HasLocalPortOrSocket() + implicit := c.IsImplicitUpstream(uid) + return upstream, !implicit && !explicit +} + type configSnapshotTerminatingGateway struct { MeshConfig *structs.MeshConfigEntry MeshConfigSet bool diff --git a/agent/service_manager_test.go b/agent/service_manager_test.go index cbbd9e5e9d..9282d1e714 100644 --- a/agent/service_manager_test.go +++ b/agent/service_manager_test.go @@ -6,10 +6,12 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "testing" "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/testrpc" @@ -419,7 +421,13 @@ func TestServiceManager_PersistService_API(t *testing.T) { "protocol": "http", }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ - structs.OpaqueUpstreamConfig{ + { + Upstream: structs.NewServiceID(structs.WildcardSpecifier, acl.DefaultEnterpriseMeta().WithWildcardNamespace()), + Config: map[string]interface{}{ + "foo": int64(1), + }, + }, + { Upstream: structs.NewServiceID("redis", nil), Config: map[string]interface{}{ "protocol": "tcp", @@ -429,7 +437,7 @@ func TestServiceManager_PersistService_API(t *testing.T) { }, EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), } - expectJSONFile(t, configFile, pcfg, resetDefaultsQueryMeta) + expectJSONFile(t, configFile, pcfg, fixPersistedServiceConfigForTest) // Verify in memory state. { @@ -467,7 +475,13 @@ func TestServiceManager_PersistService_API(t *testing.T) { "protocol": "http", }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ - structs.OpaqueUpstreamConfig{ + { + Upstream: structs.NewServiceID(structs.WildcardSpecifier, acl.DefaultEnterpriseMeta().WithWildcardNamespace()), + Config: map[string]interface{}{ + "foo": int64(1), + }, + }, + { Upstream: structs.NewServiceID("redis", nil), Config: map[string]interface{}{ "protocol": "tcp", @@ -477,7 +491,7 @@ func TestServiceManager_PersistService_API(t *testing.T) { }, EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), } - expectJSONFile(t, configFile, pcfg, resetDefaultsQueryMeta) + expectJSONFile(t, configFile, pcfg, fixPersistedServiceConfigForTest) // Verify in memory state. expectState.Proxy.LocalServicePort = 8001 @@ -646,7 +660,13 @@ func TestServiceManager_PersistService_ConfigFiles(t *testing.T) { "protocol": "http", }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ - structs.OpaqueUpstreamConfig{ + { + Upstream: structs.NewServiceID(structs.WildcardSpecifier, acl.DefaultEnterpriseMeta().WithWildcardNamespace()), + Config: map[string]interface{}{ + "foo": int64(1), + }, + }, + { Upstream: structs.NewServiceID("redis", nil), Config: map[string]interface{}{ "protocol": "tcp", @@ -655,7 +675,7 @@ func TestServiceManager_PersistService_ConfigFiles(t *testing.T) { }, }, EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, resetDefaultsQueryMeta) + }, fixPersistedServiceConfigForTest) // Verify in memory state. { @@ -813,6 +833,23 @@ func expectJSONFile(t *testing.T, file string, expect interface{}, fixupContentB require.JSONEq(t, string(expected), string(content)) } +func fixPersistedServiceConfigForTest(content []byte) ([]byte, error) { + var parsed persistedServiceConfig + if err := json.Unmarshal(content, &parsed); err != nil { + return nil, err + } + // Sort the output, since it's randomized and causes flaky tests otherwise. + sort.Slice(parsed.Defaults.UpstreamIDConfigs, func(i, j int) bool { + return parsed.Defaults.UpstreamIDConfigs[i].Upstream.ID < parsed.Defaults.UpstreamIDConfigs[j].Upstream.ID + }) + out, err := json.Marshal(parsed) + if err != nil { + return nil, err + } + // Clean the query meta + return resetDefaultsQueryMeta(out) +} + // resetDefaultsQueryMeta will reset the embedded fields from structs.QueryMeta // to their zero values in the json object keyed under 'Defaults'. func resetDefaultsQueryMeta(content []byte) ([]byte, error) { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 93e6433e6f..3e28c6b787 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -87,18 +87,10 @@ func (s *ResourceGenerator) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.C clusters = append(clusters, passthroughs...) } - getUpstream := func(uid proxycfg.UpstreamID) (*structs.Upstream, bool) { - upstream := cfgSnap.ConnectProxy.UpstreamConfig[uid] - - explicit := upstream.HasLocalPortOrSocket() - implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid) - return upstream, !implicit && !explicit - } - // NOTE: Any time we skip a chain below we MUST also skip that discovery chain in endpoints.go // so that the sets of endpoints generated matches the sets of clusters. for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { - upstream, skip := getUpstream(uid) + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) if skip { continue } @@ -123,7 +115,7 @@ func (s *ResourceGenerator) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.C // upstream in endpoints.go so that the sets of endpoints generated matches // the sets of clusters. for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { - upstream, skip := getUpstream(uid) + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) if skip { continue } diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 4251dd0194..c51e5ff913 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -50,18 +50,10 @@ func (s *ResourceGenerator) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg. cfgSnap.ConnectProxy.PeerUpstreamEndpoints.Len()+ len(cfgSnap.ConnectProxy.WatchedUpstreamEndpoints)) - getUpstream := func(uid proxycfg.UpstreamID) (*structs.Upstream, bool) { - upstream := cfgSnap.ConnectProxy.UpstreamConfig[uid] - - explicit := upstream.HasLocalPortOrSocket() - implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid) - return upstream, !implicit && !explicit - } - // NOTE: Any time we skip a chain below we MUST also skip that discovery chain in clusters.go // so that the sets of endpoints generated matches the sets of clusters. for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { - upstream, skip := getUpstream(uid) + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) if skip { // Discovery chain is not associated with a known explicit or implicit upstream so it is skipped. continue @@ -92,7 +84,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg. // upstream in clusters.go so that the sets of endpoints generated matches // the sets of clusters. for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { - upstream, skip := getUpstream(uid) + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) if skip { // Discovery chain is not associated with a known explicit or implicit upstream so it is skipped. continue @@ -555,6 +547,10 @@ func (s *ResourceGenerator) makeUpstreamLoadAssignmentForPeerService( return la, err } + if upstreamGatewayMode == structs.MeshGatewayModeNone { + s.Logger.Warn(fmt.Sprintf("invalid mesh gateway mode 'none', defaulting to 'remote' for %q", uid)) + } + // If an upstream is configured with local mesh gw mode, we make a load assignment // from the gateway endpoints instead of those of the upstreams. if upstreamGatewayMode == structs.MeshGatewayModeLocal { @@ -651,7 +647,7 @@ func (s *ResourceGenerator) endpointsFromDiscoveryChain( } mgwMode := structs.MeshGatewayModeDefault - if upstream := cfgSnap.ConnectProxy.UpstreamConfig[uid]; upstream != nil { + if upstream, _ := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta); upstream != nil { mgwMode = upstream.MeshGateway.Mode } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index ed9dfa944c..cf5ae6b567 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -126,17 +126,8 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. return nil, err } - getUpstream := func(uid proxycfg.UpstreamID) (*structs.Upstream, bool) { - upstream := cfgSnap.ConnectProxy.UpstreamConfig[uid] - - explicit := upstream.HasLocalPortOrSocket() - implicit := cfgSnap.ConnectProxy.IsImplicitUpstream(uid) - return upstream, !implicit && !explicit - } - for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { - upstreamCfg, skip := getUpstream(uid) - + upstreamCfg, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) if skip { // Discovery chain is not associated with a known explicit or implicit upstream so it is skipped. continue @@ -336,8 +327,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. // Looping over explicit and implicit upstreams is only needed for cross-peer // because they do not have discovery chains. for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { - upstreamCfg, skip := getUpstream(uid) - + upstreamCfg, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) if skip { // Not associated with a known explicit or implicit upstream so it is skipped. continue @@ -2087,10 +2077,9 @@ func (s *ResourceGenerator) getAndModifyUpstreamConfigForPeeredListener( s.Logger.Warn("failed to parse", "upstream", uid, "error", err) } - protocol := cfg.Protocol - if protocol == "" { - protocol = peerMeta.Protocol - } + // Ignore the configured protocol for peer upstreams, since it is defined by the remote + // cluster, which we cannot control. + protocol := peerMeta.Protocol if protocol == "" { protocol = "tcp" }