Add default resolvers to disco chains based on the default sameness group (#16837)

agent/configentry/discoverychain.go

@@ -18,6 +18,7 @@ type DiscoveryChainSet struct {
 	Resolvers            map[structs.ServiceID]*structs.ServiceResolverConfigEntry
 	Services             map[structs.ServiceID]*structs.ServiceConfigEntry
 	Peers                map[string]*pbpeering.Peering
+	DefaultSamenessGroup *structs.SamenessGroupConfigEntry
 	SamenessGroups       map[string]*structs.SamenessGroupConfigEntry
 	ProxyDefaults        map[string]*structs.ProxyConfigEntry
 }
@@ -69,6 +70,10 @@ func (e *DiscoveryChainSet) GetSamenessGroup(name string) *structs.SamenessGroup
 	return nil
 }
 
+func (e *DiscoveryChainSet) GetDefaultSamenessGroup() *structs.SamenessGroupConfigEntry {
+	return e.DefaultSamenessGroup
+}
+
 func (e *DiscoveryChainSet) GetProxyDefaults(partition string) *structs.ProxyConfigEntry {
 	if e.ProxyDefaults != nil {
 		return e.ProxyDefaults[partition]
@@ -116,9 +121,9 @@ func (e *DiscoveryChainSet) AddServices(entries ...*structs.ServiceConfigEntry)
 	}
 }
 
-// AddSamenessGroup adds service configs. Convenience function for testing.
+// AddSamenessGroup adds a sameness group. Convenience function for testing.
 func (e *DiscoveryChainSet) AddSamenessGroup(entries ...*structs.SamenessGroupConfigEntry) {
-	if e.Services == nil {
+	if e.SamenessGroups == nil {
 		e.SamenessGroups = make(map[string]*structs.SamenessGroupConfigEntry)
 	}
 	for _, entry := range entries {
@@ -126,6 +131,20 @@ func (e *DiscoveryChainSet) AddSamenessGroup(entries ...*structs.SamenessGroupCo
 	}
 }
 
+// SetDefaultSamenessGroup sets the default sameness group. Convenience function for testing.
+func (e *DiscoveryChainSet) SetDefaultSamenessGroup(entry *structs.SamenessGroupConfigEntry) {
+	if e.SamenessGroups == nil {
+		e.SamenessGroups = make(map[string]*structs.SamenessGroupConfigEntry)
+	}
+
+	if entry == nil {
+		return
+	}
+
+	e.SamenessGroups[entry.Name] = entry
+	e.DefaultSamenessGroup = entry
+}
+
 // AddProxyDefaults adds proxy-defaults configs. Convenience function for testing.
 func (e *DiscoveryChainSet) AddProxyDefaults(entries ...*structs.ProxyConfigEntry) {
 	if e.ProxyDefaults == nil {
@@ -149,23 +168,26 @@ func (e *DiscoveryChainSet) AddPeers(entries ...*pbpeering.Peering) {
 // AddEntries adds generic configs. Convenience function for testing. Panics on
 // operator error.
 func (e *DiscoveryChainSet) AddEntries(entries ...structs.ConfigEntry) {
-	for _, entry := range entries {
+	for _, rawEntry := range entries {
-		switch entry.GetKind() {
+		switch entry := rawEntry.(type) {
-		case structs.ServiceRouter:
+		case *structs.ServiceRouterConfigEntry:
-			e.AddRouters(entry.(*structs.ServiceRouterConfigEntry))
+			e.AddRouters(entry)
-		case structs.ServiceSplitter:
+		case *structs.ServiceSplitterConfigEntry:
-			e.AddSplitters(entry.(*structs.ServiceSplitterConfigEntry))
+			e.AddSplitters(entry)
-		case structs.ServiceResolver:
+		case *structs.ServiceResolverConfigEntry:
-			e.AddResolvers(entry.(*structs.ServiceResolverConfigEntry))
+			e.AddResolvers(entry)
-		case structs.ServiceDefaults:
+		case *structs.ServiceConfigEntry:
-			e.AddServices(entry.(*structs.ServiceConfigEntry))
+			e.AddServices(entry)
-		case structs.SamenessGroup:
+		case *structs.SamenessGroupConfigEntry:
-			e.AddSamenessGroup(entry.(*structs.SamenessGroupConfigEntry))
+			if entry.DefaultForFailover {
-		case structs.ProxyDefaults:
+				e.DefaultSamenessGroup = entry
+			}
+			e.AddSamenessGroup(entry)
+		case *structs.ProxyConfigEntry:
 			if entry.GetName() != structs.ProxyConfigGlobal {
 				panic("the only supported proxy-defaults name is '" + structs.ProxyConfigGlobal + "'")
 			}
-			e.AddProxyDefaults(entry.(*structs.ProxyConfigEntry))
+			e.AddProxyDefaults(entry)
 		default:
 			panic("unhandled config entry kind: " + entry.GetKind())
 		}
@@ -182,5 +204,5 @@ func (e *DiscoveryChainSet) IsEmpty() bool {
 // service-splitters, or service-resolvers that are present. These config
 // entries are the primary parts of the discovery chain.
 func (e *DiscoveryChainSet) IsChainEmpty() bool {
-	return len(e.Routers) == 0 && len(e.Splitters) == 0 && len(e.Resolvers) == 0
+	return len(e.Routers) == 0 && len(e.Splitters) == 0 && len(e.Resolvers) == 0 && e.DefaultSamenessGroup == nil
 }

agent/consul/discoverychain/compile.go

@@ -582,7 +582,7 @@ func (c *compiler) assembleChain() error {
 	// Check for short circuit path.
 	if len(c.resolvers) == 0 && c.entries.IsChainEmpty() {
 		// Materialize defaults and cache.
-		c.resolvers[sid] = newDefaultServiceResolver(sid)
+		c.resolvers[sid] = c.newDefaultServiceResolver(sid, "")
 	}
 
 	// The only router we consult is the one for the service name at the top of
@@ -923,7 +923,7 @@ RESOLVE_AGAIN:
 	resolver, ok := c.resolvers[targetID]
 	if !ok {
 		// Materialize defaults and cache.
-		resolver = newDefaultServiceResolver(targetID)
+		resolver = c.newDefaultServiceResolver(targetID, target.Peer)
 		c.resolvers[targetID] = resolver
 	}
 
@@ -1095,7 +1095,8 @@ RESOLVE_AGAIN:
 	}
 
 	if resolver.Redirect != nil && resolver.Redirect.SamenessGroup != "" {
-		opts := resolver.Redirect.ToDiscoveryTargetOpts()
+		opts := structs.MergeDiscoveryTargetOpts(resolver.ToSamenessDiscoveryTargetOpts(),
+			resolver.Redirect.ToDiscoveryTargetOpts())
 		failoverTargets, err = c.makeSamenessGroupFailover(target, opts, resolver.Redirect.SamenessGroup)
 		if err != nil {
 			return nil, err
@@ -1138,7 +1139,9 @@ RESOLVE_AGAIN:
 				}
 			}
 		} else if failover.SamenessGroup != "" {
-			failoverTargets, err = c.makeSamenessGroupFailover(target, failover.ToDiscoveryTargetOpts(), failover.SamenessGroup)
+			opts := structs.MergeDiscoveryTargetOpts(resolver.ToSamenessDiscoveryTargetOpts(),
+				failover.ToDiscoveryTargetOpts())
+			failoverTargets, err = c.makeSamenessGroupFailover(target, opts, failover.SamenessGroup)
 			if err != nil {
 				return nil, err
 			}
@@ -1199,7 +1202,23 @@ func (c *compiler) makeSamenessGroupFailover(target *structs.DiscoveryTarget, op
 	return failoverTargets, nil
 }
 
-func newDefaultServiceResolver(sid structs.ServiceID) *structs.ServiceResolverConfigEntry {
+func (c *compiler) newDefaultServiceResolver(sid structs.ServiceID, peer string) *structs.ServiceResolverConfigEntry {
+	sg := c.entries.GetDefaultSamenessGroup()
+	entMeta := c.GetEnterpriseMeta()
+	if sg != nil && peer == "" && (entMeta == nil || sid.PartitionOrDefault() == entMeta.PartitionOrDefault()) {
+		return &structs.ServiceResolverConfigEntry{
+			Kind:           structs.ServiceResolver,
+			Name:           sid.ID,
+			EnterpriseMeta: sid.EnterpriseMeta,
+			// This needs to be a redirect rather than failover because failovers
+			// implicitly include the local service. This isn't the behavior we want
+			// for services on sameness groups the local partition isn't a member of.
+			Redirect: &structs.ServiceResolverRedirect{
+				SamenessGroup: sg.Name,
+			},
+		}
+	}
+
 	return &structs.ServiceResolverConfigEntry{
 		Kind:           structs.ServiceResolver,
 		Name:           sid.ID,

agent/consul/state/config_entry.go

@@ -751,14 +751,70 @@ func validateProposedConfigEntryInServiceGraph(
 
 	case structs.SamenessGroup:
 		// Any service resolver could reference a sameness group.
-		_, entries, err := configEntriesByKindTxn(tx, nil, structs.ServiceResolver, wildcardEntMeta)
+		_, resolverEntries, err := configEntriesByKindTxn(tx, nil, structs.ServiceResolver, wildcardEntMeta)
 		if err != nil {
 			return err
 		}
-		for _, entry := range entries {
+		for _, entry := range resolverEntries {
 			checkChains[structs.NewServiceID(entry.GetName(), entry.GetEnterpriseMeta())] = struct{}{}
 		}
 
+		// This is the case for deleting a config entry
+		if newEntry == nil {
+			break
+		}
+
+		entry := newEntry.(*structs.SamenessGroupConfigEntry)
+
+		_, samenessGroupEntries, err := configEntriesByKindTxn(tx, nil, structs.SamenessGroup, wildcardEntMeta)
+		if err != nil {
+			return err
+		}
+
+		// Replace the existing sameness group if one exists.
+		var exists bool
+		for i := range samenessGroupEntries {
+			sg := samenessGroupEntries[i]
+			if sg.GetName() == entry.Name {
+				samenessGroupEntries[i] = entry
+				exists = true
+				break
+			}
+		}
+
+		// If this sameness group doesn't currently exist, add it.
+		if !exists {
+			samenessGroupEntries = append(samenessGroupEntries, entry)
+		}
+
+		existingPartitions := make(map[string]string)
+		existingPeers := make(map[string]string)
+		for _, e := range samenessGroupEntries {
+			sg, ok := e.(*structs.SamenessGroupConfigEntry)
+			if !ok {
+				return fmt.Errorf("type %T is not a sameness group config entry", e)
+			}
+
+			for _, m := range sg.AllMembers() {
+				if m.Peer != "" {
+					if prev, ok := existingPeers[m.Peer]; ok {
+						return fmt.Errorf("members can only belong to a single sameness group, but cluster peer %q is shared between groups %q and %q",
+							m.Peer, prev, sg.Name,
+						)
+					}
+					existingPeers[m.Peer] = sg.Name
+					continue
+				}
+
+				if prev, ok := existingPartitions[m.Partition]; ok {
+					return fmt.Errorf("members can only belong to a single sameness group, but partition %q is shared between groups %q and %q",
+						m.Partition, prev, sg.Name,
+					)
+				}
+				existingPartitions[m.Partition] = sg.Name
+			}
+		}
+
 	case structs.ProxyDefaults:
 		// Check anything that has a discovery chain entry. In the future we could
 		// somehow omit the ones that have a default protocol configured.
@@ -1499,14 +1555,29 @@ func readDiscoveryChainConfigEntriesTxn(
 			continue
 		}
 
-		for _, e := range entry.Members {
+		for _, peer := range entry.RelatedPeers() {
-			if e.Peer != "" {
+			todoPeers[peer] = struct{}{}
-				todoPeers[e.Peer] = struct{}{}
-			}
 		}
 		res.SamenessGroups[sg] = entry
 	}
 
+	if r := res.Resolvers[sid]; r == nil {
+		idx, sg, err := getDefaultSamenessGroup(tx, ws, sid.PartitionOrDefault())
+		if err != nil {
+			return 0, nil, err
+		}
+		if idx > maxIdx {
+			maxIdx = idx
+		}
+		if sg != nil {
+			res.DefaultSamenessGroup = sg
+			res.SamenessGroups[sg.Name] = sg
+			for _, peer := range sg.RelatedPeers() {
+				todoPeers[peer] = struct{}{}
+			}
+		}
+	}
+
 	for peerName := range todoPeers {
 		q := Query{
 			Value:          peerName,

agent/consul/state/config_entry_sameness_group_oss.go

@@ -14,7 +14,7 @@ import (
 	"github.com/hashicorp/go-memdb"
 )
 
-// SamnessGroupDefaultIndex is a placeholder for OSS. Sameness-groups are enterprise only.
+// SamenessGroupDefaultIndex is a placeholder for OSS. Sameness-groups are enterprise only.
 type SamenessGroupDefaultIndex struct{}
 
 var _ memdb.Indexer = (*SamenessGroupDefaultIndex)(nil)
@@ -47,3 +47,7 @@ func getSamenessGroupConfigEntryTxn(
 ) (uint64, *structs.SamenessGroupConfigEntry, error) {
 	return 0, nil, nil
 }
+
+func getDefaultSamenessGroup(tx ReadTxn, ws memdb.WatchSet, partition string) (uint64, *structs.SamenessGroupConfigEntry, error) {
+	return 0, nil, nil
+}

agent/consul/state/config_entry_schema.go

@@ -17,7 +17,7 @@ const (
 	indexLink                 = "link"
 	indexIntentionLegacyID    = "intention-legacy-id"
 	indexSource               = "intention-source"
-	indexSamenessGroupDefault = "sameness-group-default"
+	indexSamenessGroupDefault = "sameness-group-default-for-failover"
 )
 
 // configTableSchema returns a new table schema used to store global

agent/proxycfg/testing_upstreams.go

@@ -61,7 +61,8 @@ func setupTestVariationConfigEntriesAndSnapshot(
 	}
 	dbChainID := structs.ChainID(dbOpts)
 	makeChainID := func(opts structs.DiscoveryTargetOpts) string {
-		return structs.ChainID(structs.MergeDiscoveryTargetOpts(dbOpts, opts))
+		finalOpts := structs.MergeDiscoveryTargetOpts(dbOpts, opts)
+		return structs.ChainID(finalOpts)
 	}
 
 	switch variation {

agent/structs/config_entry_discoverychain.go

@@ -1367,6 +1367,16 @@ type ServiceResolverRedirect struct {
 	SamenessGroup string `json:",omitempty"`
 }
 
+// ToSamenessDiscoveryTargetOpts returns the options required for sameness failover and redirects.
+// These operations should preserve the service name and namespace.
+func (r *ServiceResolverConfigEntry) ToSamenessDiscoveryTargetOpts() DiscoveryTargetOpts {
+	return DiscoveryTargetOpts{
+		Service:   r.Name,
+		Namespace: r.NamespaceOrDefault(),
+		Partition: r.PartitionOrDefault(),
+	}
+}
+
 func (r *ServiceResolverRedirect) ToDiscoveryTargetOpts() DiscoveryTargetOpts {
 	return DiscoveryTargetOpts{
 		Service:       r.Service,

agent/structs/config_entry_sameness_group.go

@@ -12,7 +12,8 @@ import (
 
 type SamenessGroupConfigEntry struct {
 	Name               string
-	IsDefault          bool `json:",omitempty" alias:"is_default"`
+	DefaultForFailover bool `json:",omitempty" alias:"default_for_failover"`
+	IncludeLocal       bool `json:",omitempty" alias:"include_local"`
 	Members            []SamenessGroupMember
 	Meta               map[string]string `json:",omitempty"`
 	acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"`
@@ -73,20 +74,3 @@ type SamenessGroupMember struct {
 	Partition string
 	Peer      string
 }
-
-func (s *SamenessGroupConfigEntry) ToFailoverTargets() []ServiceResolverFailoverTarget {
-	if s == nil {
-		return nil
-	}
-
-	var targets []ServiceResolverFailoverTarget
-
-	for _, m := range s.Members {
-		targets = append(targets, ServiceResolverFailoverTarget{
-			Peer:      m.Peer,
-			Partition: m.Partition,
-		})
-	}
-
-	return targets
-}

agent/structs/config_entry_sameness_group_oss.go

@@ -11,3 +11,17 @@ import "fmt"
 func (s *SamenessGroupConfigEntry) Validate() error {
 	return fmt.Errorf("sameness-groups are an enterprise-only feature")
 }
+
+// RelatedPeers returns all peers that are members of a sameness group config entry.
+func (s *SamenessGroupConfigEntry) RelatedPeers() []string {
+	return nil
+}
+
+// AllMembers adds the local partition to Members when it is set.
+func (s *SamenessGroupConfigEntry) AllMembers() []SamenessGroupMember {
+	return nil
+}
+
+func (s *SamenessGroupConfigEntry) ToFailoverTargets() []ServiceResolverFailoverTarget {
+	return nil
+}

agent/structs/discovery_chain.go

@@ -266,32 +266,37 @@ type DiscoveryTargetOpts struct {
 	Peer          string
 }
 
-func MergeDiscoveryTargetOpts(o1 DiscoveryTargetOpts, o2 DiscoveryTargetOpts) DiscoveryTargetOpts {
+func MergeDiscoveryTargetOpts(opts ...DiscoveryTargetOpts) DiscoveryTargetOpts {
-	if o2.Service != "" {
+	var final DiscoveryTargetOpts
-		o1.Service = o2.Service
+	for _, o := range opts {
+		if o.Service != "" {
+			final.Service = o.Service
 		}
 
-	if o2.ServiceSubset != "" {
+		if o.ServiceSubset != "" {
-		o1.ServiceSubset = o2.ServiceSubset
+			final.ServiceSubset = o.ServiceSubset
 		}
 
-	if o2.Namespace != "" {
+		// default should override the existing value
-		o1.Namespace = o2.Namespace
+		if o.Namespace != "" {
+			final.Namespace = o.Namespace
 		}
 
-	if o2.Partition != "" {
+		// default should override the existing value
-		o1.Partition = o2.Partition
+		if o.Partition != "" {
+			final.Partition = o.Partition
 		}
 
-	if o2.Datacenter != "" {
+		if o.Datacenter != "" {
-		o1.Datacenter = o2.Datacenter
+			final.Datacenter = o.Datacenter
 		}
 
-	if o2.Peer != "" {
+		if o.Peer != "" {
-		o1.Peer = o2.Peer
+			final.Peer = o.Peer
+		}
 	}
 
-	return o1
+	return final
 }
 
 func NewDiscoveryTarget(opts DiscoveryTargetOpts) *DiscoveryTarget {

api/config_entry_sameness_group.go

@@ -7,7 +7,8 @@ type SamenessGroupConfigEntry struct {
 	Kind               string
 	Name               string
 	Partition          string `json:",omitempty"`
-	IsDefault   bool   `json:",omitempty" alias:"is_default"`
+	DefaultForFailover bool   `json:",omitempty" alias:"default_for_failover"`
+	IncludeLocal       bool   `json:",omitempty" alias:"include_local"`
 	Members            []SamenessGroupMember
 	Meta               map[string]string `json:",omitempty"`
 	CreateIndex        uint64