Merge branch 'main' into cm-bug-11457

.changelog/20873.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+gateways: api-gateway can leverage listener TLS certificates available on the gateway's local filesystem by specifying the public certificate and private key path in the new file-system-certificate configuration entry
+```

.changelog/20899.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+dns: DNS-over-grpc when using `consul-dataplane` now accepts partition, namespace, token as metadata to default those query parameters.
+`consul-dataplane` v1.5+ will send this information automatically.
+```

.changelog/20945.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+gateways: service defaults configuration entries can now be used to set default upstream limits for mesh-gateways
+```

.changelog/20977.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+server: fix Ent snapshot restore on CE when CE downgrade is enabled
+```

.changelog/_9142.txt

@@ -0,0 +1,3 @@
+```release-note: improvement
+telemetry: Add `telemetry.disable_per_tenancy_usage_metrics` in agent configuration to disable setting tenancy labels on usage metrics. This significantly decreases CPU utilization in clusters with many admin partitions or namespaces.
+```

agent/config/builder.go

@@ -947,6 +947,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 			CirconusSubmissionInterval:         stringVal(c.Telemetry.CirconusSubmissionInterval),
 			CirconusSubmissionURL:              stringVal(c.Telemetry.CirconusSubmissionURL),
 			DisableHostname:                    boolVal(c.Telemetry.DisableHostname),
+			DisablePerTenancyUsageMetrics:      boolVal(c.Telemetry.DisablePerTenancyUsageMetrics),
 			DogstatsdAddr:                      stringVal(c.Telemetry.DogstatsdAddr),
 			DogstatsdTags:                      c.Telemetry.DogstatsdTags,
 			RetryFailedConfiguration:           boolVal(c.Telemetry.RetryFailedConfiguration),

agent/config/config.go

@@ -697,6 +697,7 @@ type Telemetry struct {
 	CirconusSubmissionInterval         *string  `mapstructure:"circonus_submission_interval" json:"circonus_submission_interval,omitempty"`
 	CirconusSubmissionURL              *string  `mapstructure:"circonus_submission_url" json:"circonus_submission_url,omitempty"`
 	DisableHostname                    *bool    `mapstructure:"disable_hostname" json:"disable_hostname,omitempty"`
+	DisablePerTenancyUsageMetrics      *bool    `mapstructure:"disable_per_tenancy_usage_metrics" json:"disable_per_tenancy_usage_metrics,omitempty"`
 	EnableHostMetrics                  *bool    `mapstructure:"enable_host_metrics" json:"enable_host_metrics,omitempty"`
 	DogstatsdAddr                      *string  `mapstructure:"dogstatsd_addr" json:"dogstatsd_addr,omitempty"`
 	DogstatsdTags                      []string `mapstructure:"dogstatsd_tags" json:"dogstatsd_tags,omitempty"`

agent/config/runtime_test.go

@@ -6023,7 +6023,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) {
 		json: []string{`
 			{
 				"experiments": ["resource-apis"]
-			}	
+			}
 		`},
 		hcl: []string{`experiments=["resource-apis"]`},
 		expected: func(rt *RuntimeConfig) {
@@ -6974,7 +6974,8 @@ func TestLoad_FullConfig(t *testing.T) {
 				Expiration: 15 * time.Second,
 				Name:       "ftO6DySn", // notice this is the same as the metrics prefix
 			},
-			EnableHostMetrics: true,
+			EnableHostMetrics:             true,
+			DisablePerTenancyUsageMetrics: true,
 		},
 		TLS: tlsutil.Config{
 			InternalRPC: tlsutil.ProtocolConfig{

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -208,11 +208,11 @@
     "ExposeMaxPort": 0,
     "ExposeMinPort": 0,
     "GRPCAddrs": [],
+    "GRPCKeepaliveInterval": "0s",
+    "GRPCKeepaliveTimeout": "0s",
     "GRPCPort": 0,
     "GRPCTLSAddrs": [],
     "GRPCTLSPort": 0,
-    "GRPCKeepaliveInterval": "0s",
-    "GRPCKeepaliveTimeout": "0s",
     "GossipLANGossipInterval": "0s",
     "GossipLANGossipNodes": 0,
     "GossipLANProbeInterval": "0s",
@@ -472,6 +472,7 @@
         "CirconusSubmissionURL": "",
         "Disable": false,
         "DisableHostname": false,
+        "DisablePerTenancyUsageMetrics": false,
         "DogstatsdAddr": "",
         "DogstatsdTags": [],
         "EnableHostMetrics": false,

agent/config/testdata/full-config.hcl

@@ -718,6 +718,7 @@ telemetry {
     prometheus_retention_time = "15s"
     statsd_address = "drce87cy"
     statsite_address = "HpFwKB8R"
+    disable_per_tenancy_usage_metrics = true
 }
 tls {
     defaults {

agent/config/testdata/full-config.json

@@ -841,7 +841,8 @@
     "metrics_prefix": "ftO6DySn",
     "prometheus_retention_time": "15s",
     "statsd_address": "drce87cy",
-    "statsite_address": "HpFwKB8R"
+    "statsite_address": "HpFwKB8R",
+    "disable_per_tenancy_usage_metrics": true
   },
   "tls": {
     "defaults": {
@@ -944,4 +945,4 @@
   "xds": {
     "update_max_per_second": 9526.2
   }
-}
+}

agent/consul/config.go

@@ -404,6 +404,8 @@ type Config struct {
 	// report usage metrics to the configured go-metrics Sinks.
 	MetricsReportingInterval time.Duration
 
+	DisablePerTenancyUsageMetrics bool
+
 	// ConnectEnabled is whether to enable Connect features such as the CA.
 	ConnectEnabled bool
 

agent/consul/fsm/decode_downgrade.go

@@ -619,6 +619,8 @@ func MakeShadowConfigEntry(kind, name string) (structs.ConfigEntry, error) {
 		return &ShadowAPIGatewayConfigEntry{APIGatewayConfigEntry: &structs.APIGatewayConfigEntry{Name: name}}, nil
 	case structs.BoundAPIGateway:
 		return &ShadowBoundAPIGatewayConfigEntry{BoundAPIGatewayConfigEntry: &structs.BoundAPIGatewayConfigEntry{Name: name}}, nil
+	case structs.FileSystemCertificate:
+		return &ShadowFileSystemCertificateConfigEntry{FileSystemCertificateConfigEntry: &structs.FileSystemCertificateConfigEntry{Name: name}}, nil
 	case structs.InlineCertificate:
 		return &ShadowInlineCertificateConfigEntry{InlineCertificateConfigEntry: &structs.InlineCertificateConfigEntry{Name: name}}, nil
 	case structs.HTTPRoute:
@@ -931,6 +933,15 @@ func (s ShadowBoundAPIGatewayConfigEntry) GetRealConfigEntry() structs.ConfigEnt
 	return s.BoundAPIGatewayConfigEntry
 }
 
+type ShadowFileSystemCertificateConfigEntry struct {
+	ShadowBase
+	*structs.FileSystemCertificateConfigEntry
+}
+
+func (s ShadowFileSystemCertificateConfigEntry) GetRealConfigEntry() structs.ConfigEntry {
+	return s.FileSystemCertificateConfigEntry
+}
+
 type ShadowInlineCertificateConfigEntry struct {
 	ShadowBase
 	*structs.InlineCertificateConfigEntry

agent/consul/fsm/fsm.go

@@ -196,7 +196,7 @@ func (c *FSM) Apply(log *raft.Log) interface{} {
 		return nil
 	}
 	if structs.CEDowngrade && msgType >= 64 {
-		c.logger.Warn("ignoring enterprise message, for downgrading to oss", "type", msgType)
+		c.logger.Warn("ignoring enterprise message as part of downgrade to CE", "type", msgType)
 		return nil
 	}
 	panic(fmt.Errorf("failed to apply request: %#v", buf))
@@ -268,8 +268,9 @@ func (c *FSM) Restore(old io.ReadCloser) error {
 			}
 		default:
 			if structs.CEDowngrade && msg >= 64 {
-				c.logger.Warn("ignoring enterprise message , for downgrading to oss", "type", msg)
-				return nil
+				c.logger.Warn("ignoring enterprise message as part of downgrade to CE", "type", msg)
+				var ignore interface{}
+				return dec.Decode(&ignore)
 			} else if msg >= 64 {
 				return fmt.Errorf("msg type <%d> is a Consul Enterprise log entry. Consul CE cannot restore it", msg)
 			} else {
@@ -402,6 +403,11 @@ func (c *FSM) registerStreamSnapshotHandlers() {
 	}, true)
 	panicIfErr(err)
 
+	err = c.deps.Publisher.RegisterHandler(state.EventTopicFileSystemCertificate, func(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
+		return c.State().FileSystemCertificateSnapshot(req, buf)
+	}, true)
+	panicIfErr(err)
+
 	err = c.deps.Publisher.RegisterHandler(state.EventTopicInlineCertificate, func(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
 		return c.State().InlineCertificateSnapshot(req, buf)
 	}, true)

agent/consul/fsm/snapshot_ce_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/hashicorp/consul/agent/consul/state"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/lib/stringslice"
 	"github.com/hashicorp/consul/sdk/testutil"
 )
 
@@ -60,3 +61,101 @@ func TestRestoreFromEnterprise(t *testing.T) {
 	require.EqualError(t, fsm.Restore(sink), "msg type <65> is a Consul Enterprise log entry. Consul CE cannot restore it")
 	sink.Cancel()
 }
+
+func TestRestoreFromEnterprise_CEDowngrade(t *testing.T) {
+	logger := testutil.Logger(t)
+
+	handle := &testRaftHandle{}
+	storageBackend := newStorageBackend(t, handle)
+	handle.apply = func(buf []byte) (any, error) { return storageBackend.Apply(buf, 123), nil }
+
+	fsm := NewFromDeps(Deps{
+		Logger: logger,
+		NewStateStore: func() *state.Store {
+			return state.NewStateStore(nil)
+		},
+		StorageBackend: storageBackend,
+	})
+
+	// To verify if a proper message is displayed when Consul CE tries to
+	//  unsuccessfully restore entries from a Consul Ent snapshot.
+	buf := bytes.NewBuffer(nil)
+	sink := &MockSink{buf, false}
+
+	type EntMock struct {
+		ID   int
+		Type string
+	}
+
+	entMockEntry := EntMock{
+		ID:   65,
+		Type: "A Consul Ent Log Type",
+	}
+
+	// Create one entry to exercise the Go struct marshaller, and one to exercise the
+	// Binary Marshaller interface. This verifies that regardless of whether the struct gets
+	// encoded as a msgpack byte string (binary marshaller) or msgpack map (other struct),
+	// it will still be skipped over correctly.
+	registerEntry := structs.RegisterRequest{
+		Datacenter: "dc1",
+		Node:       "foo",
+		Address:    "127.0.0.1",
+		Service: &structs.NodeService{
+			ID:      "db",
+			Service: "db",
+			Tags:    []string{"primary"},
+			Port:    8000,
+		},
+	}
+	proxyDefaultsEntry := &structs.ConfigEntryRequest{
+		Op: structs.ConfigEntryUpsert,
+		Entry: &structs.ProxyConfigEntry{
+			Kind: structs.ProxyDefaults,
+			Name: "global",
+			Config: map[string]interface{}{
+				"foo": "bar",
+			},
+			EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(),
+		},
+	}
+
+	// Write the header and records.
+	header := SnapshotHeader{
+		LastIndex: 0,
+	}
+	encoder := codec.NewEncoder(sink, structs.MsgpackHandle)
+	encoder.Encode(&header)
+	sink.Write([]byte{byte(structs.MessageType(entMockEntry.ID))})
+	encoder.Encode(entMockEntry)
+	sink.Write([]byte{byte(structs.RegisterRequestType)})
+	encoder.Encode(registerEntry)
+	sink.Write([]byte{byte(structs.ConfigEntryRequestType)})
+	encoder.Encode(proxyDefaultsEntry)
+
+	defer func() {
+		structs.CEDowngrade = false
+	}()
+	structs.CEDowngrade = true
+
+	require.NoError(t, fsm.Restore(sink), "failed to decode Ent snapshot to CE")
+
+	// Verify the register request
+	_, nodes, err := fsm.state.Nodes(nil, nil, "")
+	require.NoError(t, err)
+	require.Len(t, nodes, 1, "incorrect number of nodes: %v", nodes)
+	require.Equal(t, "foo", nodes[0].Node)
+	require.Equal(t, "dc1", nodes[0].Datacenter)
+	require.Equal(t, "127.0.0.1", nodes[0].Address)
+	_, fooSrv, err := fsm.state.NodeServices(nil, "foo", nil, "")
+	require.NoError(t, err)
+	require.Len(t, fooSrv.Services, 1)
+	require.Contains(t, fooSrv.Services["db"].Tags, "primary")
+	require.True(t, stringslice.Contains(fooSrv.Services["db"].Tags, "primary"))
+	require.Equal(t, 8000, fooSrv.Services["db"].Port)
+
+	// Verify the proxy defaults request
+	_, configEntry, err := fsm.state.ConfigEntry(nil, structs.ProxyDefaults, "global", structs.DefaultEnterpriseMetaInDefaultPartition())
+	require.NoError(t, err)
+	configEntry.SetHash(proxyDefaultsEntry.Entry.GetHash())
+	require.Equal(t, proxyDefaultsEntry.Entry, configEntry)
+}

agent/consul/gateways/controller_gateways.go

@@ -61,7 +61,7 @@ func (r *apiGatewayReconciler) Reconcile(ctx context.Context, req controller.Req
 		return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileHTTPRoute, r.cleanupRoute)
 	case structs.TCPRoute:
 		return reconcileEntry(r.fsm.State(), r.logger, ctx, req, r.reconcileTCPRoute, r.cleanupRoute)
-	case structs.InlineCertificate:
+	case structs.InlineCertificate, structs.FileSystemCertificate:
 		return r.enqueueCertificateReferencedGateways(r.fsm.State(), ctx, req)
 	case structs.JWTProvider:
 		return r.enqueueJWTProviderReferencedGatewaysAndHTTPRoutes(r.fsm.State(), ctx, req)

agent/consul/server.go

@@ -854,6 +854,7 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server,
 			WithStateProvider(s.fsm).
 			WithLogger(s.logger).
 			WithDatacenter(s.config.Datacenter).
+			WithDisabledTenancyMetrics(s.config.DisablePerTenancyUsageMetrics).
 			WithReportingInterval(s.config.MetricsReportingInterval).
 			WithGetMembersFunc(func() []serf.Member {
 				members, err := s.lanPoolAllMembers()

agent/consul/state/config_entry.go

@@ -646,6 +646,7 @@ func validateProposedConfigEntryInGraph(
 	case structs.ExportedServices:
 	case structs.APIGateway: // TODO Consider checkGatewayClash
 	case structs.BoundAPIGateway:
+	case structs.FileSystemCertificate:
 	case structs.InlineCertificate:
 	case structs.HTTPRoute:
 	case structs.TCPRoute:

agent/consul/state/config_entry_events.go

@@ -15,20 +15,21 @@ import (
 
 // Adding events for a new config entry kind? Remember to update ConfigEntryFromStructs and ConfigEntryToStructs.
 var configEntryKindToTopic = map[string]stream.Topic{
-	structs.MeshConfig:        EventTopicMeshConfig,
-	structs.ServiceResolver:   EventTopicServiceResolver,
-	structs.IngressGateway:    EventTopicIngressGateway,
-	structs.ServiceIntentions: EventTopicServiceIntentions,
-	structs.ServiceDefaults:   EventTopicServiceDefaults,
-	structs.APIGateway:        EventTopicAPIGateway,
-	structs.TCPRoute:          EventTopicTCPRoute,
-	structs.HTTPRoute:         EventTopicHTTPRoute,
-	structs.InlineCertificate: EventTopicInlineCertificate,
-	structs.BoundAPIGateway:   EventTopicBoundAPIGateway,
-	structs.RateLimitIPConfig: EventTopicIPRateLimit,
-	structs.SamenessGroup:     EventTopicSamenessGroup,
-	structs.JWTProvider:       EventTopicJWTProvider,
-	structs.ExportedServices:  EventTopicExportedServices,
+	structs.MeshConfig:            EventTopicMeshConfig,
+	structs.ServiceResolver:       EventTopicServiceResolver,
+	structs.IngressGateway:        EventTopicIngressGateway,
+	structs.ServiceIntentions:     EventTopicServiceIntentions,
+	structs.ServiceDefaults:       EventTopicServiceDefaults,
+	structs.APIGateway:            EventTopicAPIGateway,
+	structs.TCPRoute:              EventTopicTCPRoute,
+	structs.HTTPRoute:             EventTopicHTTPRoute,
+	structs.FileSystemCertificate: EventTopicFileSystemCertificate,
+	structs.InlineCertificate:     EventTopicInlineCertificate,
+	structs.BoundAPIGateway:       EventTopicBoundAPIGateway,
+	structs.RateLimitIPConfig:     EventTopicIPRateLimit,
+	structs.SamenessGroup:         EventTopicSamenessGroup,
+	structs.JWTProvider:           EventTopicJWTProvider,
+	structs.ExportedServices:      EventTopicExportedServices,
 }
 
 // EventSubjectConfigEntry is a stream.Subject used to route and receive events
@@ -147,6 +148,12 @@ func (s *Store) HTTPRouteSnapshot(req stream.SubscribeRequest, buf stream.Snapsh
 	return s.configEntrySnapshot(structs.HTTPRoute, req, buf)
 }
 
+// FileSystemCertificateSnapshot is a stream.SnapshotFunc that returns a snapshot of
+// inline-certificate config entries.
+func (s *Store) FileSystemCertificateSnapshot(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {
+	return s.configEntrySnapshot(structs.FileSystemCertificate, req, buf)
+}
+
 // InlineCertificateSnapshot is a stream.SnapshotFunc that returns a snapshot of
 // inline-certificate config entries.
 func (s *Store) InlineCertificateSnapshot(req stream.SubscribeRequest, buf stream.SnapshotAppender) (uint64, error) {

agent/consul/state/events.go

@@ -46,7 +46,8 @@ func PBToStreamSubscribeRequest(req *pbsubscribe.SubscribeRequest, entMeta acl.E
 		case EventTopicMeshConfig, EventTopicServiceResolver, EventTopicIngressGateway,
 			EventTopicServiceIntentions, EventTopicServiceDefaults, EventTopicAPIGateway,
 			EventTopicTCPRoute, EventTopicHTTPRoute, EventTopicJWTProvider, EventTopicInlineCertificate,
-			EventTopicBoundAPIGateway, EventTopicSamenessGroup, EventTopicExportedServices:
+			EventTopicBoundAPIGateway, EventTopicSamenessGroup, EventTopicExportedServices,
+			EventTopicFileSystemCertificate:
 			subject = EventSubjectConfigEntry{
 				Name:           named.Key,
 				EnterpriseMeta: &entMeta,

agent/consul/state/memdb.go

@@ -196,23 +196,24 @@ func (db *readDB) ReadTxn() AbortTxn {
 }
 
 var (
-	EventTopicServiceHealth        = pbsubscribe.Topic_ServiceHealth
-	EventTopicServiceHealthConnect = pbsubscribe.Topic_ServiceHealthConnect
-	EventTopicMeshConfig           = pbsubscribe.Topic_MeshConfig
-	EventTopicServiceResolver      = pbsubscribe.Topic_ServiceResolver
-	EventTopicIngressGateway       = pbsubscribe.Topic_IngressGateway
-	EventTopicServiceIntentions    = pbsubscribe.Topic_ServiceIntentions
-	EventTopicServiceDefaults      = pbsubscribe.Topic_ServiceDefaults
-	EventTopicServiceList          = pbsubscribe.Topic_ServiceList
-	EventTopicAPIGateway           = pbsubscribe.Topic_APIGateway
-	EventTopicTCPRoute             = pbsubscribe.Topic_TCPRoute
-	EventTopicHTTPRoute            = pbsubscribe.Topic_HTTPRoute
-	EventTopicInlineCertificate    = pbsubscribe.Topic_InlineCertificate
-	EventTopicBoundAPIGateway      = pbsubscribe.Topic_BoundAPIGateway
-	EventTopicIPRateLimit          = pbsubscribe.Topic_IPRateLimit
-	EventTopicSamenessGroup        = pbsubscribe.Topic_SamenessGroup
-	EventTopicJWTProvider          = pbsubscribe.Topic_JWTProvider
-	EventTopicExportedServices     = pbsubscribe.Topic_ExportedServices
+	EventTopicServiceHealth         = pbsubscribe.Topic_ServiceHealth
+	EventTopicServiceHealthConnect  = pbsubscribe.Topic_ServiceHealthConnect
+	EventTopicMeshConfig            = pbsubscribe.Topic_MeshConfig
+	EventTopicServiceResolver       = pbsubscribe.Topic_ServiceResolver
+	EventTopicIngressGateway        = pbsubscribe.Topic_IngressGateway
+	EventTopicServiceIntentions     = pbsubscribe.Topic_ServiceIntentions
+	EventTopicServiceDefaults       = pbsubscribe.Topic_ServiceDefaults
+	EventTopicServiceList           = pbsubscribe.Topic_ServiceList
+	EventTopicAPIGateway            = pbsubscribe.Topic_APIGateway
+	EventTopicTCPRoute              = pbsubscribe.Topic_TCPRoute
+	EventTopicHTTPRoute             = pbsubscribe.Topic_HTTPRoute
+	EventTopicFileSystemCertificate = pbsubscribe.Topic_FileSystemCertificate
+	EventTopicInlineCertificate     = pbsubscribe.Topic_InlineCertificate
+	EventTopicBoundAPIGateway       = pbsubscribe.Topic_BoundAPIGateway
+	EventTopicIPRateLimit           = pbsubscribe.Topic_IPRateLimit
+	EventTopicSamenessGroup         = pbsubscribe.Topic_SamenessGroup
+	EventTopicJWTProvider           = pbsubscribe.Topic_JWTProvider
+	EventTopicExportedServices      = pbsubscribe.Topic_ExportedServices
 )
 
 func processDBChanges(tx ReadTxn, changes Changes) ([]stream.Event, error) {

agent/consul/usagemetrics/usagemetrics.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/serf/serf"
 
 	"github.com/hashicorp/consul/agent/consul/state"
+	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/logging"
 	"github.com/hashicorp/consul/version"
 )
@@ -76,6 +77,7 @@ type Config struct {
 	stateProvider  StateProvider
 	tickerInterval time.Duration
 	getMembersFunc getMembersFunc
+	excludeTenancy bool
 }
 
 // WithDatacenter adds the datacenter as a label to all metrics emitted by the
@@ -85,6 +87,12 @@ func (c *Config) WithDatacenter(dc string) *Config {
 	return c
 }
 
+// WithDisabledTenancyMetrics opts the user out of specifying usage metrics for each tenancy.
+func (c *Config) WithDisabledTenancyMetrics(disabled bool) *Config {
+	c.excludeTenancy = disabled
+	return c
+}
+
 // WithLogger takes a logger and creates a new, named sub-logger to use when
 // running
 func (c *Config) WithLogger(logger hclog.Logger) *Config {
@@ -125,6 +133,9 @@ type UsageMetricsReporter struct {
 	stateProvider  StateProvider
 	tickerInterval time.Duration
 	getMembersFunc getMembersFunc
+	excludeTenancy bool
+
+	usageReporter
 }
 
 func NewUsageMetricsReporter(cfg *Config) (*UsageMetricsReporter, error) {
@@ -151,8 +162,11 @@ func NewUsageMetricsReporter(cfg *Config) (*UsageMetricsReporter, error) {
 		metricLabels:   cfg.metricLabels,
 		tickerInterval: cfg.tickerInterval,
 		getMembersFunc: cfg.getMembersFunc,
+		excludeTenancy: cfg.excludeTenancy,
 	}
 
+	u.usageReporter = newTenancyUsageReporter(u)
+
 	return u, nil
 }
 
@@ -190,7 +204,7 @@ func (u *UsageMetricsReporter) runOnce() {
 
 	u.emitPeeringUsage(peeringUsage)
 
-	_, serviceUsage, err := state.ServiceUsage(nil, true)
+	_, serviceUsage, err := state.ServiceUsage(nil, !u.excludeTenancy)
 	if err != nil {
 		u.logger.Warn("failed to retrieve services from state store", "error", err)
 	}
@@ -259,3 +273,112 @@ func versionWithMetadata() string {
 
 	return vsn
 }
+
+type usageReporter interface {
+	emitNodeUsage(nodeUsage state.NodeUsage)
+	emitPeeringUsage(peeringUsage state.PeeringUsage)
+	emitMemberUsage(members []serf.Member)
+	emitServiceUsage(serviceUsage structs.ServiceUsage)
+	emitKVUsage(kvUsage state.KVUsage)
+	emitConfigEntryUsage(configUsage state.ConfigEntryUsage)
+}
+
+type baseUsageReporter struct {
+	metricLabels []metrics.Label
+}
+
+var _ usageReporter = (*baseUsageReporter)(nil)
+
+func newBaseUsageReporter(u *UsageMetricsReporter) *baseUsageReporter {
+	return &baseUsageReporter{
+		metricLabels: u.metricLabels,
+	}
+}
+
+func (u *baseUsageReporter) emitNodeUsage(nodeUsage state.NodeUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "nodes"},
+		float32(nodeUsage.Nodes),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitPeeringUsage(peeringUsage state.PeeringUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "peerings"},
+		float32(peeringUsage.Peerings),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitMemberUsage(members []serf.Member) {
+	var (
+		servers int
+		clients int
+	)
+	for _, m := range members {
+		switch m.Tags["role"] {
+		case "node":
+			clients++
+		case "consul":
+			servers++
+		}
+	}
+
+	metrics.SetGaugeWithLabels(
+		[]string{"members", "clients"},
+		float32(clients),
+		u.metricLabels,
+	)
+
+	metrics.SetGaugeWithLabels(
+		[]string{"members", "servers"},
+		float32(servers),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitServiceUsage(serviceUsage structs.ServiceUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "services"},
+		float32(serviceUsage.Services),
+		u.metricLabels,
+	)
+
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "service_instances"},
+		float32(serviceUsage.ServiceInstances),
+		u.metricLabels,
+	)
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "billable_service_instances"},
+		float32(serviceUsage.BillableServiceInstances),
+		u.metricLabels,
+	)
+
+	for k, i := range serviceUsage.ConnectServiceInstances {
+		metrics.SetGaugeWithLabels(
+			[]string{"state", "connect_instances"},
+			float32(i),
+			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
+		)
+	}
+}
+
+func (u *baseUsageReporter) emitKVUsage(kvUsage state.KVUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "kv_entries"},
+		float32(kvUsage.KVCount),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitConfigEntryUsage(configUsage state.ConfigEntryUsage) {
+	for k, i := range configUsage.ConfigByKind {
+		metrics.SetGaugeWithLabels(
+			[]string{"state", "config_entries"},
+			float32(i),
+			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
+		)
+	}
+}

agent/consul/usagemetrics/usagemetrics_ce.go

@@ -5,99 +5,6 @@
 
 package usagemetrics
 
-import (
-	"github.com/armon/go-metrics"
-
-	"github.com/hashicorp/serf/serf"
-
-	"github.com/hashicorp/consul/agent/consul/state"
-	"github.com/hashicorp/consul/agent/structs"
-)
-
-func (u *UsageMetricsReporter) emitNodeUsage(nodeUsage state.NodeUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "nodes"},
-		float32(nodeUsage.Nodes),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitPeeringUsage(peeringUsage state.PeeringUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "peerings"},
-		float32(peeringUsage.Peerings),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitMemberUsage(members []serf.Member) {
-	var (
-		servers int
-		clients int
-	)
-	for _, m := range members {
-		switch m.Tags["role"] {
-		case "node":
-			clients++
-		case "consul":
-			servers++
-		}
-	}
-
-	metrics.SetGaugeWithLabels(
-		[]string{"members", "clients"},
-		float32(clients),
-		u.metricLabels,
-	)
-
-	metrics.SetGaugeWithLabels(
-		[]string{"members", "servers"},
-		float32(servers),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitServiceUsage(serviceUsage structs.ServiceUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "services"},
-		float32(serviceUsage.Services),
-		u.metricLabels,
-	)
-
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "service_instances"},
-		float32(serviceUsage.ServiceInstances),
-		u.metricLabels,
-	)
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "billable_service_instances"},
-		float32(serviceUsage.BillableServiceInstances),
-		u.metricLabels,
-	)
-
-	for k, i := range serviceUsage.ConnectServiceInstances {
-		metrics.SetGaugeWithLabels(
-			[]string{"state", "connect_instances"},
-			float32(i),
-			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
-		)
-	}
-}
-
-func (u *UsageMetricsReporter) emitKVUsage(kvUsage state.KVUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "kv_entries"},
-		float32(kvUsage.KVCount),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitConfigEntryUsage(configUsage state.ConfigEntryUsage) {
-	for k, i := range configUsage.ConfigByKind {
-		metrics.SetGaugeWithLabels(
-			[]string{"state", "config_entries"},
-			float32(i),
-			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
-		)
-	}
+func newTenancyUsageReporter(u *UsageMetricsReporter) usageReporter {
+	return newBaseUsageReporter(u)
 }

agent/consul/usagemetrics/usagemetrics_ce_test.go

@@ -6,938 +6,46 @@
 package usagemetrics
 
 import (
-	"fmt"
 	"testing"
 	"time"
 
 	"github.com/armon/go-metrics"
-	"github.com/hashicorp/serf/serf"
 	"github.com/stretchr/testify/require"
 
-	"github.com/hashicorp/go-uuid"
-
-	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/consul/state"
-	"github.com/hashicorp/consul/agent/structs"
-	"github.com/hashicorp/consul/proto/private/pbpeering"
 	"github.com/hashicorp/consul/sdk/testutil"
-	"github.com/hashicorp/consul/version"
 )
 
 func newStateStore() (*state.Store, error) {
 	return state.NewStateStore(nil), nil
 }
 
-type testCase struct {
-	modfiyStateStore func(t *testing.T, s *state.Store)
-	getMembersFunc   getMembersFunc
-	expectedGauges   map[string]metrics.GaugeValue
-}
+func TestUsageReporter_CE(t *testing.T) {
+	getMetricsReporter := func(tc testCase) (*UsageMetricsReporter, *metrics.InmemSink, error) {
+		// Only have a single interval for the test
+		sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
+		cfg := metrics.DefaultConfig("consul.usage.test")
+		cfg.EnableHostname = false
+		metrics.NewGlobal(cfg, sink)
 
-var baseCases = map[string]testCase{
-	"empty-state": {
-		expectedGauges: map[string]metrics.GaugeValue{
-			// --- node ---
-			"consul.usage.test.state.nodes;datacenter=dc1": {
-				Name:   "consul.usage.test.state.nodes",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- peering ---
-			"consul.usage.test.state.peerings;datacenter=dc1": {
-				Name:   "consul.usage.test.state.peerings",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- member ---
-			"consul.usage.test.members.clients;datacenter=dc1": {
-				Name:   "consul.usage.test.members.clients",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.members.servers;datacenter=dc1": {
-				Name:   "consul.usage.test.members.servers",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service ---
-			"consul.usage.test.state.services;datacenter=dc1": {
-				Name:   "consul.usage.test.state.services",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.state.service_instances;datacenter=dc1": {
-				Name:   "consul.usage.test.state.service_instances",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service mesh ---
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-proxy"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-native"},
-				},
-			},
-			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
-				Name:  "consul.usage.test.state.billable_service_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-				},
-			},
-			// --- kv ---
-			"consul.usage.test.state.kv_entries;datacenter=dc1": {
-				Name:   "consul.usage.test.state.kv_entries",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- config entries ---
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-intentions"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-resolver"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-router"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-splitter"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "proxy-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "exported-services"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "sameness-group"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "bound-api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "inline-certificate"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "http-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "tcp-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "jwt-provider"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "control-plane-request-limit"},
-				},
-			},
-			// --- version ---
-			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
-				Name:  "consul.usage.test.version",
-				Value: 1,
-				Labels: []metrics.Label{
-					{Name: "version", Value: versionWithMetadata()},
-					{Name: "pre_release", Value: version.VersionPrerelease},
-				},
-			},
-		},
-		getMembersFunc: func() []serf.Member { return []serf.Member{} },
-	},
-	"nodes": {
-		modfiyStateStore: func(t *testing.T, s *state.Store) {
-			require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
-			require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
-		},
-		getMembersFunc: func() []serf.Member {
-			return []serf.Member{
-				{
-					Name:   "foo",
-					Tags:   map[string]string{"role": "consul"},
-					Status: serf.StatusAlive,
-				},
-				{
-					Name:   "bar",
-					Tags:   map[string]string{"role": "consul"},
-					Status: serf.StatusAlive,
-				},
-			}
-		},
-		expectedGauges: map[string]metrics.GaugeValue{
-			// --- node ---
-			"consul.usage.test.state.nodes;datacenter=dc1": {
-				Name:   "consul.usage.test.state.nodes",
-				Value:  2,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- peering ---
-			"consul.usage.test.state.peerings;datacenter=dc1": {
-				Name:   "consul.usage.test.state.peerings",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- member ---
-			"consul.usage.test.members.servers;datacenter=dc1": {
-				Name:   "consul.usage.test.members.servers",
-				Value:  2,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.members.clients;datacenter=dc1": {
-				Name:   "consul.usage.test.members.clients",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service ---
-			"consul.usage.test.state.services;datacenter=dc1": {
-				Name:   "consul.usage.test.state.services",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.state.service_instances;datacenter=dc1": {
-				Name:   "consul.usage.test.state.service_instances",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service mesh ---
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-proxy"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-native"},
-				},
-			},
-			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
-				Name:  "consul.usage.test.state.billable_service_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-				},
-			},
-			// --- kv ---
-			"consul.usage.test.state.kv_entries;datacenter=dc1": {
-				Name:   "consul.usage.test.state.kv_entries",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- config entries ---
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-intentions"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-resolver"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-router"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-splitter"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "proxy-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "exported-services"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "sameness-group"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "bound-api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "inline-certificate"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "http-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "tcp-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "jwt-provider"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "control-plane-request-limit"},
-				},
-			},
-			// --- version ---
-			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
-				Name:  "consul.usage.test.version",
-				Value: 1,
-				Labels: []metrics.Label{
-					{Name: "version", Value: versionWithMetadata()},
-					{Name: "pre_release", Value: version.VersionPrerelease},
-				},
-			},
-		},
-	},
-}
-
-func TestUsageReporter_emitNodeUsage_CE(t *testing.T) {
-	cases := baseCases
-
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			// Only have a single interval for the test
-			sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
-			cfg := metrics.DefaultConfig("consul.usage.test")
-			cfg.EnableHostname = false
-			metrics.NewGlobal(cfg, sink)
-
-			mockStateProvider := &mockStateProvider{}
-			s, err := newStateStore()
-			require.NoError(t, err)
-			if tcase.modfiyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
-			}
-			mockStateProvider.On("State").Return(s)
-
-			reporter, err := NewUsageMetricsReporter(
-				new(Config).
-					WithStateProvider(mockStateProvider).
-					WithLogger(testutil.Logger(t)).
-					WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
-			)
-			require.NoError(t, err)
-
-			reporter.runOnce()
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
-	}
-}
-
-func TestUsageReporter_emitPeeringUsage_CE(t *testing.T) {
-	cases := make(map[string]testCase)
-	for k, v := range baseCases {
-		eg := make(map[string]metrics.GaugeValue)
-		for k, v := range v.expectedGauges {
-			eg[k] = v
-		}
-		cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg}
-	}
-	peeringsCase := cases["nodes"]
-	peeringsCase.modfiyStateStore = func(t *testing.T, s *state.Store) {
-		id, err := uuid.GenerateUUID()
+		mockStateProvider := &mockStateProvider{}
+		s, err := newStateStore()
 		require.NoError(t, err)
-		require.NoError(t, s.PeeringWrite(1, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "foo", ID: id}}))
-		id, err = uuid.GenerateUUID()
-		require.NoError(t, err)
-		require.NoError(t, s.PeeringWrite(2, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "bar", ID: id}}))
-		id, err = uuid.GenerateUUID()
-		require.NoError(t, err)
-		require.NoError(t, s.PeeringWrite(3, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "baz", ID: id}}))
-	}
-	peeringsCase.getMembersFunc = func() []serf.Member {
-		return []serf.Member{
-			{
-				Name:   "foo",
-				Tags:   map[string]string{"role": "consul"},
-				Status: serf.StatusAlive,
-			},
-			{
-				Name:   "bar",
-				Tags:   map[string]string{"role": "consul"},
-				Status: serf.StatusAlive,
-			},
+		if tc.modifyStateStore != nil {
+			tc.modifyStateStore(t, s)
 		}
+		mockStateProvider.On("State").Return(s)
+
+		reporter, err := NewUsageMetricsReporter(
+			new(Config).
+				WithStateProvider(mockStateProvider).
+				WithLogger(testutil.Logger(t)).
+				WithDatacenter("dc1").
+				WithGetMembersFunc(tc.getMembersFunc),
+		)
+
+		return reporter, sink, err
 	}
-	peeringsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.nodes",
-		Value:  0,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	peeringsCase.expectedGauges["consul.usage.test.state.peerings;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.peerings",
-		Value:  3,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	peeringsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.members.clients",
-		Value:  0,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	cases["peerings"] = peeringsCase
-	delete(cases, "nodes")
 
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			// Only have a single interval for the test
-			sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
-			cfg := metrics.DefaultConfig("consul.usage.test")
-			cfg.EnableHostname = false
-			metrics.NewGlobal(cfg, sink)
-
-			mockStateProvider := &mockStateProvider{}
-			s, err := newStateStore()
-			require.NoError(t, err)
-			if tcase.modfiyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
-			}
-			mockStateProvider.On("State").Return(s)
-
-			reporter, err := NewUsageMetricsReporter(
-				new(Config).
-					WithStateProvider(mockStateProvider).
-					WithLogger(testutil.Logger(t)).
-					WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
-			)
-			require.NoError(t, err)
-
-			reporter.runOnce()
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
-	}
-}
-
-func TestUsageReporter_emitServiceUsage_CE(t *testing.T) {
-	cases := make(map[string]testCase)
-	for k, v := range baseCases {
-		eg := make(map[string]metrics.GaugeValue)
-		for k, v := range v.expectedGauges {
-			eg[k] = v
-		}
-		cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg}
-	}
-
-	nodesAndSvcsCase := cases["nodes"]
-	nodesAndSvcsCase.modfiyStateStore = func(t *testing.T, s *state.Store) {
-		require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
-		require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
-		require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"}))
-		require.NoError(t, s.EnsureNode(4, &structs.Node{Node: "qux", Address: "127.0.0.3"}))
-
-		apigw := structs.TestNodeServiceAPIGateway(t)
-		apigw.ID = "api-gateway"
-
-		mgw := structs.TestNodeServiceMeshGateway(t)
-		mgw.ID = "mesh-gateway"
-
-		tgw := structs.TestNodeServiceTerminatingGateway(t, "1.1.1.1")
-		tgw.ID = "terminating-gateway"
-		// Typical services and some consul services spread across two nodes
-		require.NoError(t, s.EnsureService(5, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: nil, Address: "", Port: 5000}))
-		require.NoError(t, s.EnsureService(6, "bar", &structs.NodeService{ID: "api", Service: "api", Tags: nil, Address: "", Port: 5000}))
-		require.NoError(t, s.EnsureService(7, "foo", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
-		require.NoError(t, s.EnsureService(8, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
-		require.NoError(t, s.EnsureService(9, "foo", &structs.NodeService{ID: "db-connect-proxy", Service: "db-connect-proxy", Tags: nil, Address: "", Port: 5000, Kind: structs.ServiceKindConnectProxy}))
-		require.NoError(t, s.EnsureRegistration(10, structs.TestRegisterIngressGateway(t)))
-		require.NoError(t, s.EnsureService(11, "foo", mgw))
-		require.NoError(t, s.EnsureService(12, "foo", tgw))
-		require.NoError(t, s.EnsureService(13, "foo", apigw))
-		require.NoError(t, s.EnsureService(14, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}}))
-		require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{
-			Kind: structs.IngressGateway,
-			Name: "foo",
-		}))
-		require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{
-			Kind: structs.IngressGateway,
-			Name: "bar",
-		}))
-		require.NoError(t, s.EnsureConfigEntry(17, &structs.IngressGatewayConfigEntry{
-			Kind: structs.IngressGateway,
-			Name: "baz",
-		}))
-	}
-	baseCaseMembers := nodesAndSvcsCase.getMembersFunc()
-	nodesAndSvcsCase.getMembersFunc = func() []serf.Member {
-		baseCaseMembers = append(baseCaseMembers, serf.Member{
-			Name:   "baz",
-			Tags:   map[string]string{"role": "node", "segment": "a"},
-			Status: serf.StatusAlive,
-		})
-		baseCaseMembers = append(baseCaseMembers, serf.Member{
-			Name:   "qux",
-			Tags:   map[string]string{"role": "node", "segment": "b"},
-			Status: serf.StatusAlive,
-		})
-		return baseCaseMembers
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.nodes",
-		Value:  4,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.members.clients",
-		Value:  2,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.services;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.services",
-		Value:  8,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.service_instances",
-		Value:  10,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "connect-proxy"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "terminating-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "ingress-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "api-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "mesh-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "connect-native"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.billable_service_instances;datacenter=dc1"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.billable_service_instances",
-		Value: 3,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.config_entries",
-		Value: 3,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "ingress-gateway"},
-		},
-	}
-	cases["nodes-and-services"] = nodesAndSvcsCase
-	delete(cases, "nodes")
-
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			// Only have a single interval for the test
-			sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
-			cfg := metrics.DefaultConfig("consul.usage.test")
-			cfg.EnableHostname = false
-			metrics.NewGlobal(cfg, sink)
-
-			mockStateProvider := &mockStateProvider{}
-			s := state.NewStateStore(nil)
-			if tcase.modfiyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
-			}
-			mockStateProvider.On("State").Return(s)
-
-			reporter, err := NewUsageMetricsReporter(
-				new(Config).
-					WithStateProvider(mockStateProvider).
-					WithLogger(testutil.Logger(t)).
-					WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
-			)
-			require.NoError(t, err)
-
-			reporter.runOnce()
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
-	}
-}
-
-func TestUsageReporter_emitKVUsage_CE(t *testing.T) {
-	cases := make(map[string]testCase)
-	for k, v := range baseCases {
-		eg := make(map[string]metrics.GaugeValue)
-		for k, v := range v.expectedGauges {
-			eg[k] = v
-		}
-		cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg}
-	}
-
-	nodesCase := cases["nodes"]
-	mss := nodesCase.modfiyStateStore
-	nodesCase.modfiyStateStore = func(t *testing.T, s *state.Store) {
-		mss(t, s)
-		require.NoError(t, s.KVSSet(4, &structs.DirEntry{Key: "a", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(5, &structs.DirEntry{Key: "b", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(6, &structs.DirEntry{Key: "c", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(7, &structs.DirEntry{Key: "d", Value: []byte{1}}))
-		require.NoError(t, s.KVSDelete(8, "d", &acl.EnterpriseMeta{}))
-		require.NoError(t, s.KVSDelete(9, "c", &acl.EnterpriseMeta{}))
-		require.NoError(t, s.KVSSet(10, &structs.DirEntry{Key: "e", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(11, &structs.DirEntry{Key: "f", Value: []byte{1}}))
-	}
-	nodesCase.expectedGauges["consul.usage.test.state.kv_entries;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.kv_entries",
-		Value:  4,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	cases["nodes"] = nodesCase
-
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			// Only have a single interval for the test
-			sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
-			cfg := metrics.DefaultConfig("consul.usage.test")
-			cfg.EnableHostname = false
-			metrics.NewGlobal(cfg, sink)
-
-			mockStateProvider := &mockStateProvider{}
-			s, err := newStateStore()
-			require.NoError(t, err)
-			if tcase.modfiyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
-			}
-			mockStateProvider.On("State").Return(s)
-
-			reporter, err := NewUsageMetricsReporter(
-				new(Config).
-					WithStateProvider(mockStateProvider).
-					WithLogger(testutil.Logger(t)).
-					WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
-			)
-			require.NoError(t, err)
-
-			reporter.runOnce()
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
-	}
+	testUsageReporter_Tenantless(t, getMetricsReporter)
 }

agent/consul/usagemetrics/usagemetrics_test.go

@@ -10,15 +10,19 @@ import (
 
 	"github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/prometheus"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/serf/serf"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
+	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/consul/state"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/lib"
-	"github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/serf/serf"
+	"github.com/hashicorp/consul/proto/private/pbpeering"
+	"github.com/hashicorp/consul/version"
 )
 
 type mockStateProvider struct {
@@ -119,3 +123,872 @@ type benchStateProvider func() *state.Store
 func (b benchStateProvider) State() *state.Store {
 	return b()
 }
+
+type testCase struct {
+	modifyStateStore func(t *testing.T, s *state.Store)
+	getMembersFunc   getMembersFunc
+	expectedGauges   map[string]metrics.GaugeValue
+}
+
+var baseCases = map[string]testCase{
+	"empty-state": {
+		expectedGauges: map[string]metrics.GaugeValue{
+			// --- node ---
+			"consul.usage.test.state.nodes;datacenter=dc1": {
+				Name:   "consul.usage.test.state.nodes",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- peering ---
+			"consul.usage.test.state.peerings;datacenter=dc1": {
+				Name:   "consul.usage.test.state.peerings",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- member ---
+			"consul.usage.test.members.clients;datacenter=dc1": {
+				Name:   "consul.usage.test.members.clients",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.members.servers;datacenter=dc1": {
+				Name:   "consul.usage.test.members.servers",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service ---
+			"consul.usage.test.state.services;datacenter=dc1": {
+				Name:   "consul.usage.test.state.services",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.state.service_instances;datacenter=dc1": {
+				Name:   "consul.usage.test.state.service_instances",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service mesh ---
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-proxy"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-native"},
+				},
+			},
+			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
+				Name:  "consul.usage.test.state.billable_service_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+				},
+			},
+			// --- kv ---
+			"consul.usage.test.state.kv_entries;datacenter=dc1": {
+				Name:   "consul.usage.test.state.kv_entries",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- config entries ---
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-intentions"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-resolver"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-router"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-splitter"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "proxy-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "exported-services"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "sameness-group"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "bound-api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=file-system-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "file-system-certificate"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "inline-certificate"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "http-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "tcp-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "jwt-provider"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "control-plane-request-limit"},
+				},
+			},
+			// --- version ---
+			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
+				Name:  "consul.usage.test.version",
+				Value: 1,
+				Labels: []metrics.Label{
+					{Name: "version", Value: versionWithMetadata()},
+					{Name: "pre_release", Value: version.VersionPrerelease},
+				},
+			},
+		},
+		getMembersFunc: func() []serf.Member { return []serf.Member{} },
+	},
+	"nodes": {
+		modifyStateStore: func(t *testing.T, s *state.Store) {
+			require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
+			require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
+		},
+		getMembersFunc: func() []serf.Member {
+			return []serf.Member{
+				{
+					Name:   "foo",
+					Tags:   map[string]string{"role": "consul"},
+					Status: serf.StatusAlive,
+				},
+				{
+					Name:   "bar",
+					Tags:   map[string]string{"role": "consul"},
+					Status: serf.StatusAlive,
+				},
+			}
+		},
+		expectedGauges: map[string]metrics.GaugeValue{
+			// --- node ---
+			"consul.usage.test.state.nodes;datacenter=dc1": {
+				Name:   "consul.usage.test.state.nodes",
+				Value:  2,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- peering ---
+			"consul.usage.test.state.peerings;datacenter=dc1": {
+				Name:   "consul.usage.test.state.peerings",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- member ---
+			"consul.usage.test.members.servers;datacenter=dc1": {
+				Name:   "consul.usage.test.members.servers",
+				Value:  2,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.members.clients;datacenter=dc1": {
+				Name:   "consul.usage.test.members.clients",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service ---
+			"consul.usage.test.state.services;datacenter=dc1": {
+				Name:   "consul.usage.test.state.services",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.state.service_instances;datacenter=dc1": {
+				Name:   "consul.usage.test.state.service_instances",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service mesh ---
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-proxy"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-native"},
+				},
+			},
+			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
+				Name:  "consul.usage.test.state.billable_service_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+				},
+			},
+			// --- kv ---
+			"consul.usage.test.state.kv_entries;datacenter=dc1": {
+				Name:   "consul.usage.test.state.kv_entries",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- config entries ---
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-intentions"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-resolver"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-router"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-splitter"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "proxy-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "exported-services"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "sameness-group"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "bound-api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=file-system-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "file-system-certificate"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "inline-certificate"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "http-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "tcp-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "jwt-provider"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "control-plane-request-limit"},
+				},
+			},
+			// --- version ---
+			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
+				Name:  "consul.usage.test.version",
+				Value: 1,
+				Labels: []metrics.Label{
+					{Name: "version", Value: versionWithMetadata()},
+					{Name: "pre_release", Value: version.VersionPrerelease},
+				},
+			},
+		},
+	},
+}
+
+func testUsageReporter_Tenantless(t *testing.T, getMetricsReporter func(tc testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	t.Run("emitNodeUsage", func(t *testing.T) {
+		testUsageReporter_emitNodeUsage_CE(t, getMetricsReporter)
+	})
+
+	t.Run("emitPeeringUsage", func(t *testing.T) {
+		testUsageReporter_emitPeeringUsage_CE(t, getMetricsReporter)
+	})
+
+	t.Run("emitServiceUsage", func(t *testing.T) {
+		testUsageReporter_emitServiceUsage_CE(t, getMetricsReporter)
+	})
+
+	t.Run("emitKVUsage", func(t *testing.T) {
+		testUsageReporter_emitKVUsage_CE(t, getMetricsReporter)
+	})
+}
+
+func testUsageReporter_emitNodeUsage_CE(t *testing.T, getReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := baseCases
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}
+
+func testUsageReporter_emitPeeringUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := make(map[string]testCase)
+	for k, v := range baseCases {
+		eg := make(map[string]metrics.GaugeValue)
+		for k, v := range v.expectedGauges {
+			eg[k] = v
+		}
+		cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg}
+	}
+	peeringsCase := cases["nodes"]
+	peeringsCase.modifyStateStore = func(t *testing.T, s *state.Store) {
+		id, err := uuid.GenerateUUID()
+		require.NoError(t, err)
+		require.NoError(t, s.PeeringWrite(1, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "foo", ID: id}}))
+		id, err = uuid.GenerateUUID()
+		require.NoError(t, err)
+		require.NoError(t, s.PeeringWrite(2, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "bar", ID: id}}))
+		id, err = uuid.GenerateUUID()
+		require.NoError(t, err)
+		require.NoError(t, s.PeeringWrite(3, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "baz", ID: id}}))
+	}
+	peeringsCase.getMembersFunc = func() []serf.Member {
+		return []serf.Member{
+			{
+				Name:   "foo",
+				Tags:   map[string]string{"role": "consul"},
+				Status: serf.StatusAlive,
+			},
+			{
+				Name:   "bar",
+				Tags:   map[string]string{"role": "consul"},
+				Status: serf.StatusAlive,
+			},
+		}
+	}
+	peeringsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.nodes",
+		Value:  0,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	peeringsCase.expectedGauges["consul.usage.test.state.peerings;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.peerings",
+		Value:  3,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	peeringsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.members.clients",
+		Value:  0,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	cases["peerings"] = peeringsCase
+	delete(cases, "nodes")
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getMetricsReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}
+
+func testUsageReporter_emitServiceUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := make(map[string]testCase)
+	for k, v := range baseCases {
+		eg := make(map[string]metrics.GaugeValue)
+		for k, v := range v.expectedGauges {
+			eg[k] = v
+		}
+		cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg}
+	}
+
+	nodesAndSvcsCase := cases["nodes"]
+	nodesAndSvcsCase.modifyStateStore = func(t *testing.T, s *state.Store) {
+		require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
+		require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
+		require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"}))
+		require.NoError(t, s.EnsureNode(4, &structs.Node{Node: "qux", Address: "127.0.0.3"}))
+
+		apigw := structs.TestNodeServiceAPIGateway(t)
+		apigw.ID = "api-gateway"
+
+		mgw := structs.TestNodeServiceMeshGateway(t)
+		mgw.ID = "mesh-gateway"
+
+		tgw := structs.TestNodeServiceTerminatingGateway(t, "1.1.1.1")
+		tgw.ID = "terminating-gateway"
+		// Typical services and some consul services spread across two nodes
+		require.NoError(t, s.EnsureService(5, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: nil, Address: "", Port: 5000}))
+		require.NoError(t, s.EnsureService(6, "bar", &structs.NodeService{ID: "api", Service: "api", Tags: nil, Address: "", Port: 5000}))
+		require.NoError(t, s.EnsureService(7, "foo", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
+		require.NoError(t, s.EnsureService(8, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
+		require.NoError(t, s.EnsureService(9, "foo", &structs.NodeService{ID: "db-connect-proxy", Service: "db-connect-proxy", Tags: nil, Address: "", Port: 5000, Kind: structs.ServiceKindConnectProxy}))
+		require.NoError(t, s.EnsureRegistration(10, structs.TestRegisterIngressGateway(t)))
+		require.NoError(t, s.EnsureService(11, "foo", mgw))
+		require.NoError(t, s.EnsureService(12, "foo", tgw))
+		require.NoError(t, s.EnsureService(13, "foo", apigw))
+		require.NoError(t, s.EnsureService(14, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}}))
+		require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{
+			Kind: structs.IngressGateway,
+			Name: "foo",
+		}))
+		require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{
+			Kind: structs.IngressGateway,
+			Name: "bar",
+		}))
+		require.NoError(t, s.EnsureConfigEntry(17, &structs.IngressGatewayConfigEntry{
+			Kind: structs.IngressGateway,
+			Name: "baz",
+		}))
+	}
+	baseCaseMembers := nodesAndSvcsCase.getMembersFunc()
+	nodesAndSvcsCase.getMembersFunc = func() []serf.Member {
+		baseCaseMembers = append(baseCaseMembers, serf.Member{
+			Name:   "baz",
+			Tags:   map[string]string{"role": "node", "segment": "a"},
+			Status: serf.StatusAlive,
+		})
+		baseCaseMembers = append(baseCaseMembers, serf.Member{
+			Name:   "qux",
+			Tags:   map[string]string{"role": "node", "segment": "b"},
+			Status: serf.StatusAlive,
+		})
+		return baseCaseMembers
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.nodes",
+		Value:  4,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.members.clients",
+		Value:  2,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.services;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.services",
+		Value:  8,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.service_instances",
+		Value:  10,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "connect-proxy"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "terminating-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "ingress-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "api-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "mesh-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "connect-native"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.billable_service_instances;datacenter=dc1"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.billable_service_instances",
+		Value: 3,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.config_entries",
+		Value: 3,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "ingress-gateway"},
+		},
+	}
+	cases["nodes-and-services"] = nodesAndSvcsCase
+	delete(cases, "nodes")
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getMetricsReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}
+
+func testUsageReporter_emitKVUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := make(map[string]testCase)
+	for k, v := range baseCases {
+		eg := make(map[string]metrics.GaugeValue)
+		for k, v := range v.expectedGauges {
+			eg[k] = v
+		}
+		cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg}
+	}
+
+	nodesCase := cases["nodes"]
+	mss := nodesCase.modifyStateStore
+	nodesCase.modifyStateStore = func(t *testing.T, s *state.Store) {
+		mss(t, s)
+		require.NoError(t, s.KVSSet(4, &structs.DirEntry{Key: "a", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(5, &structs.DirEntry{Key: "b", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(6, &structs.DirEntry{Key: "c", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(7, &structs.DirEntry{Key: "d", Value: []byte{1}}))
+		require.NoError(t, s.KVSDelete(8, "d", &acl.EnterpriseMeta{}))
+		require.NoError(t, s.KVSDelete(9, "c", &acl.EnterpriseMeta{}))
+		require.NoError(t, s.KVSSet(10, &structs.DirEntry{Key: "e", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(11, &structs.DirEntry{Key: "f", Value: []byte{1}}))
+	}
+	nodesCase.expectedGauges["consul.usage.test.state.kv_entries;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.kv_entries",
+		Value:  4,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	cases["nodes"] = nodesCase
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getMetricsReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}

agent/discovery/query_fetcher_v1.go

@@ -116,11 +116,17 @@ func (f *V1DataFetcher) FetchNodes(ctx Context, req *QueryPayload) ([]*Result, e
 		// Nodes are not namespaced, so this is a name error
 		return nil, ErrNotFound
 	}
-
 	cfg := f.dynamicConfig.Load().(*V1DataFetcherDynamicConfig)
+
+	// If no datacenter is passed, default to our own
+	datacenter := cfg.Datacenter
+	if req.Tenancy.Datacenter != "" {
+		datacenter = req.Tenancy.Datacenter
+	}
+
 	// Make an RPC request
 	args := &structs.NodeSpecificRequest{
-		Datacenter: req.Tenancy.Datacenter,
+		Datacenter: datacenter,
 		PeerName:   req.Tenancy.Peer,
 		Node:       req.Name,
 		QueryOptions: structs.QueryOptions{
@@ -299,9 +305,15 @@ func (f *V1DataFetcher) FetchWorkload(ctx Context, req *QueryPayload) (*Result,
 func (f *V1DataFetcher) FetchPreparedQuery(ctx Context, req *QueryPayload) ([]*Result, error) {
 	cfg := f.dynamicConfig.Load().(*V1DataFetcherDynamicConfig)
 
+	// If no datacenter is passed, default to our own
+	datacenter := cfg.Datacenter
+	if req.Tenancy.Datacenter != "" {
+		datacenter = req.Tenancy.Datacenter
+	}
+
 	// Execute the prepared query.
 	args := structs.PreparedQueryExecuteRequest{
-		Datacenter:    req.Tenancy.Datacenter,
+		Datacenter:    datacenter,
 		QueryIDOrName: req.Name,
 		QueryOptions: structs.QueryOptions{
 			Token:      ctx.Token,
@@ -548,7 +560,11 @@ func (f *V1DataFetcher) fetchServiceBasedOnTenancy(ctx Context, req *QueryPayloa
 		return nil, errors.New("sameness groups are not allowed for service lookups based on tenancy")
 	}
 
-	datacenter := req.Tenancy.Datacenter
+	// If no datacenter is passed, default to our own
+	datacenter := cfg.Datacenter
+	if req.Tenancy.Datacenter != "" {
+		datacenter = req.Tenancy.Datacenter
+	}
 	if req.Tenancy.Peer != "" {
 		datacenter = ""
 	}

agent/discovery/query_fetcher_v1_test.go

@@ -128,6 +128,7 @@ func Test_FetchVirtualIP(t *testing.T) {
 func Test_FetchEndpoints(t *testing.T) {
 	// set these to confirm that RPC call does not use them for this particular RPC
 	rc := &config.RuntimeConfig{
+		Datacenter:     "dc2",
 		DNSAllowStale:  true,
 		DNSMaxStale:    100,
 		DNSUseCache:    true,

agent/discovery/query_fetcher_v2.go

@@ -347,6 +347,7 @@ func queryTenancyToResourceTenancy(qTenancy QueryTenancy) *pbresource.Tenancy {
 		rTenancy.Namespace = qTenancy.Namespace
 	}
 	// In the case of partition, we have the agent's partition as the fallback.
+	// That is handled in NormalizeRequest.
 	if qTenancy.Partition != "" {
 		rTenancy.Partition = qTenancy.Partition
 	}

agent/dns/context.go

@@ -0,0 +1,56 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package dns
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/mitchellh/mapstructure"
+	"google.golang.org/grpc/metadata"
+)
+
+// Context is used augment a DNS message with Consul-specific metadata.
+type Context struct {
+	Token            string `mapstructure:"x-consul-token,omitempty"`
+	DefaultNamespace string `mapstructure:"x-consul-namespace,omitempty"`
+	DefaultPartition string `mapstructure:"x-consul-partition,omitempty"`
+}
+
+// NewContextFromGRPCContext returns the request context using the gRPC metadata attached to the
+// given context. If there is no gRPC metadata, it returns an empty context.
+func NewContextFromGRPCContext(ctx context.Context) (Context, error) {
+	if ctx == nil {
+		return Context{}, nil
+	}
+
+	reqCtx := Context{}
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return reqCtx, nil
+	}
+
+	m := map[string]string{}
+	for k, v := range md {
+		m[k] = v[0]
+	}
+
+	decoderConfig := &mapstructure.DecoderConfig{
+		Metadata:         nil,
+		Result:           &reqCtx,
+		WeaklyTypedInput: true,
+	}
+
+	decoder, err := mapstructure.NewDecoder(decoderConfig)
+	if err != nil {
+		return Context{}, fmt.Errorf("error creating mapstructure decoder: %w", err)
+	}
+
+	err = decoder.Decode(m)
+	if err != nil {
+		return Context{}, fmt.Errorf("error decoding metadata: %w", err)
+	}
+
+	return reqCtx, nil
+}

agent/dns/context_test.go

@@ -0,0 +1,70 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package dns
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestNewContextFromGRPCContext(t *testing.T) {
+	t.Parallel()
+
+	md := metadata.MD{}
+	testMeta := map[string]string{
+		"x-consul-token":     "test-token",
+		"x-consul-namespace": "test-namespace",
+		"x-consul-partition": "test-partition",
+	}
+
+	for k, v := range testMeta {
+		md.Set(k, v)
+	}
+	testGRPCContext := metadata.NewIncomingContext(context.Background(), md)
+
+	testCases := []struct {
+		name     string
+		grpcCtx  context.Context
+		expected *Context
+		error    error
+	}{
+		{
+			name:     "nil grpc context",
+			grpcCtx:  nil,
+			expected: &Context{},
+		},
+		{
+			name:     "grpc context w/o metadata",
+			grpcCtx:  context.Background(),
+			expected: &Context{},
+		},
+		{
+			name:    "grpc context w/ kitchen sink",
+			grpcCtx: testGRPCContext,
+			expected: &Context{
+				Token:            "test-token",
+				DefaultNamespace: "test-namespace",
+				DefaultPartition: "test-partition",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, err := NewContextFromGRPCContext(tc.grpcCtx)
+			if tc.error != nil {
+				require.Error(t, err)
+				require.Equal(t, Context{}, &ctx)
+				require.Equal(t, tc.error, err)
+				return
+			}
+
+			require.NotNil(t, ctx)
+			require.Equal(t, tc.expected, &ctx)
+		})
+	}
+}

agent/dns/discovery_results_fetcher.go

@@ -206,20 +206,24 @@ func getQueryTenancy(reqCtx Context, queryType discovery.QueryType, querySuffixe
 		return discovery.QueryTenancy{}, errNameNotFound
 	}
 
-	// If we don't have an explicit partition in the request, try the first fallback
+	// If we don't have an explicit partition/ns in the request, try the first fallback
 	// which was supplied in the request context. The agent's partition will be used as the last fallback
 	// later in the query processor.
 	if labels.Partition == "" {
 		labels.Partition = reqCtx.DefaultPartition
 	}
 
+	if labels.Namespace == "" {
+		labels.Namespace = reqCtx.DefaultNamespace
+	}
+
 	// If we have a sameness group, we can return early without further data massage.
 	if labels.SamenessGroup != "" {
 		return discovery.QueryTenancy{
 			Namespace:     labels.Namespace,
 			Partition:     labels.Partition,
 			SamenessGroup: labels.SamenessGroup,
-			Datacenter:    reqCtx.DefaultDatacenter,
+			// Datacenter is not supported
 		}, nil
 	}
 
@@ -234,19 +238,19 @@ func getQueryTenancy(reqCtx Context, queryType discovery.QueryType, querySuffixe
 		Namespace:  labels.Namespace,
 		Partition:  labels.Partition,
 		Peer:       labels.Peer,
-		Datacenter: getEffectiveDatacenter(labels, reqCtx.DefaultDatacenter),
+		Datacenter: getEffectiveDatacenter(labels),
 	}, nil
 }
 
 // getEffectiveDatacenter returns the effective datacenter from the parsed labels.
-func getEffectiveDatacenter(labels *parsedLabels, defaultDC string) string {
+func getEffectiveDatacenter(labels *parsedLabels) string {
 	switch {
 	case labels.Datacenter != "":
 		return labels.Datacenter
 	case labels.PeerOrDatacenter != "" && labels.Peer != labels.PeerOrDatacenter:
 		return labels.PeerOrDatacenter
 	}
-	return defaultDC
+	return ""
 }
 
 // getQueryTypePartsAndSuffixesFromDNSMessage returns the query type, the parts, and suffixes of the query name.

agent/dns/discovery_results_fetcher_test.go

@@ -160,8 +160,7 @@ func Test_buildQueryFromDNSMessage(t *testing.T) {
 				},
 			},
 			requestContext: &Context{
-				DefaultDatacenter: "default-dc",
-				DefaultPartition:  "default-partition",
+				DefaultPartition: "default-partition",
 			},
 			expectedQuery: &discovery.Query{
 				QueryType: discovery.QueryTypeWorkload,
@@ -169,10 +168,9 @@ func Test_buildQueryFromDNSMessage(t *testing.T) {
 					Name:     "foo",
 					PortName: "api",
 					Tenancy: discovery.QueryTenancy{
-						Namespace:  "banana",
-						Partition:  "orange",
-						Peer:       "apple",
-						Datacenter: "default-dc",
+						Namespace: "banana",
+						Partition: "orange",
+						Peer:      "apple",
 					},
 				},
 			},
@@ -192,8 +190,7 @@ func Test_buildQueryFromDNSMessage(t *testing.T) {
 				},
 			},
 			requestContext: &Context{
-				DefaultDatacenter: "default-dc",
-				DefaultPartition:  "default-partition",
+				DefaultPartition: "default-partition",
 			},
 			expectedQuery: &discovery.Query{
 				QueryType: discovery.QueryTypeService,
@@ -203,7 +200,6 @@ func Test_buildQueryFromDNSMessage(t *testing.T) {
 						Namespace:     "banana",
 						Partition:     "orange",
 						SamenessGroup: "apple",
-						Datacenter:    "default-dc",
 					},
 				},
 			},

agent/dns/router.go

@@ -14,9 +14,8 @@ import (
 
 	"github.com/armon/go-metrics"
 	"github.com/armon/go-radix"
-	"github.com/miekg/dns"
-
 	"github.com/hashicorp/go-hclog"
+	"github.com/miekg/dns"
 
 	"github.com/hashicorp/consul/agent/config"
 	"github.com/hashicorp/consul/agent/discovery"
@@ -46,13 +45,6 @@ var (
 	trailingSpacesRE = regexp.MustCompile(" +$")
 )
 
-// Context is used augment a DNS message with Consul-specific metadata.
-type Context struct {
-	Token             string
-	DefaultPartition  string
-	DefaultDatacenter string
-}
-
 // RouterDynamicConfig is the dynamic configuration that can be hot-reloaded
 type RouterDynamicConfig struct {
 	ARecordLimit       int
@@ -114,13 +106,12 @@ type dnsRecursor interface {
 // Router replaces miekg/dns.ServeMux with a simpler router that only checks for the 2-3 valid domains
 // that Consul supports and forwards to a single DiscoveryQueryProcessor handler. If there is no match, it will recurse.
 type Router struct {
-	processor  DiscoveryQueryProcessor
-	recursor   dnsRecursor
-	domain     string
-	altDomain  string
-	datacenter string
-	nodeName   string
-	logger     hclog.Logger
+	processor DiscoveryQueryProcessor
+	recursor  dnsRecursor
+	domain    string
+	altDomain string
+	nodeName  string
+	logger    hclog.Logger
 
 	tokenFunc                   func() string
 	translateAddressFunc        func(dc string, addr string, taggedAddresses map[string]string, accept dnsutil.TranslateAddressAccept) string
@@ -146,7 +137,6 @@ func NewRouter(cfg Config) (*Router, error) {
 		recursor:                    newRecursor(logger),
 		domain:                      domain,
 		altDomain:                   altDomain,
-		datacenter:                  cfg.AgentConfig.Datacenter,
 		logger:                      logger,
 		nodeName:                    cfg.AgentConfig.NodeName,
 		tokenFunc:                   cfg.TokenFunc,
@@ -176,6 +166,7 @@ func (r *Router) HandleRequest(req *dns.Msg, reqCtx Context, remoteAddress net.A
 	}
 
 	r.logger.Trace("received request", "question", req.Question[0].Name, "type", dns.Type(req.Question[0].Qtype).String())
+	r.normalizeContext(&reqCtx)
 
 	defer func(s time.Time, q dns.Question) {
 		metrics.MeasureSinceWithLabels([]string{"dns", "query"}, s,
@@ -319,10 +310,9 @@ func (r *Router) trimDomain(questionName string) string {
 }
 
 // ServeDNS implements the miekg/dns.Handler interface.
-// This is a standard DNS listener, so we inject a default request context based on the agent's config.
+// This is a standard DNS listener.
 func (r *Router) ServeDNS(w dns.ResponseWriter, req *dns.Msg) {
-	reqCtx := r.defaultAgentDNSRequestContext()
-	out := r.HandleRequest(req, reqCtx, w.RemoteAddr())
+	out := r.HandleRequest(req, Context{}, w.RemoteAddr())
 	w.WriteMsg(out)
 }
 
@@ -420,16 +410,6 @@ func (r *Router) GetConfig() *RouterDynamicConfig {
 	return r.dynamicConfig.Load().(*RouterDynamicConfig)
 }
 
-// defaultAgentDNSRequestContext returns a default request context based on the agent's config.
-func (r *Router) defaultAgentDNSRequestContext() Context {
-	return Context{
-		Token:             r.tokenFunc(),
-		DefaultDatacenter: r.datacenter,
-		// We don't need to specify the agent's partition here because that will be handled further down the stack
-		// in the query processor.
-	}
-}
-
 // getErrorFromECSNotGlobalError returns the underlying error from an ECSNotGlobalError, if it exists.
 func getErrorFromECSNotGlobalError(err error) error {
 	if errors.Is(err, discovery.ErrECSNotGlobal) {
@@ -471,6 +451,16 @@ func validateAndNormalizeRequest(req *dns.Msg) error {
 	return nil
 }
 
+// normalizeContext makes sure context information is populated with agent defaults as needed.
+// Right now this is just the ACL token. We do this in the router with the token because DNS doesn't
+// allow a token to be passed in the request, and we expect ACL tokens upfront in APIs when they are enabled.
+// Tenancy information is left out because it is safe/expected to assume agent defaults in the backend lookup.
+func (r *Router) normalizeContext(ctx *Context) {
+	if ctx.Token == "" {
+		ctx.Token = r.tokenFunc()
+	}
+}
+
 // stripAnyFailoverSuffix strips off the suffixes that may have been added to the request name.
 func stripAnyFailoverSuffix(target string) (string, bool) {
 	enableFailover := false

agent/dns/router_test.go

@@ -52,22 +52,6 @@ type HandleTestCase struct {
 	response                     *dns.Msg
 }
 
-var testSOA = &dns.SOA{
-	Hdr: dns.RR_Header{
-		Name:   "consul.",
-		Rrtype: dns.TypeSOA,
-		Class:  dns.ClassINET,
-		Ttl:    4,
-	},
-	Ns:      "ns.consul.",
-	Mbox:    "hostmaster.consul.",
-	Serial:  uint32(time.Now().Unix()),
-	Refresh: 1,
-	Retry:   2,
-	Expire:  3,
-	Minttl:  4,
-}
-
 func Test_HandleRequest_Validation(t *testing.T) {
 	testCases := []HandleTestCase{
 		{
@@ -93,6 +77,157 @@ func Test_HandleRequest_Validation(t *testing.T) {
 				Extra:    nil,
 			},
 		},
+		// Context Tests
+		{
+			name: "When a request context is provided, use those field in the query",
+			request: &dns.Msg{
+				MsgHdr: dns.MsgHdr{
+					Opcode: dns.OpcodeQuery,
+				},
+				Question: []dns.Question{
+					{
+						Name:   "foo.service.consul.",
+						Qtype:  dns.TypeA,
+						Qclass: dns.ClassINET,
+					},
+				},
+			},
+			requestContext: &Context{
+				Token:            "test-token",
+				DefaultNamespace: "test-namespace",
+				DefaultPartition: "test-partition",
+			},
+			configureDataFetcher: func(fetcher discovery.CatalogDataFetcher) {
+				result := []*discovery.Result{
+					{
+						Type: discovery.ResultTypeNode,
+						Node: &discovery.Location{Name: "foo", Address: "1.2.3.4"},
+						Tenancy: discovery.ResultTenancy{
+							Namespace: "test-namespace",
+							Partition: "test-partition",
+						},
+					},
+				}
+
+				fetcher.(*discovery.MockCatalogDataFetcher).
+					On("FetchEndpoints", mock.Anything, mock.Anything, mock.Anything).
+					Return(result, nil).
+					Run(func(args mock.Arguments) {
+						ctx := args.Get(0).(discovery.Context)
+						req := args.Get(1).(*discovery.QueryPayload)
+						reqType := args.Get(2).(discovery.LookupType)
+
+						require.Equal(t, "test-token", ctx.Token)
+
+						require.Equal(t, "foo", req.Name)
+						require.Equal(t, "test-namespace", req.Tenancy.Namespace)
+						require.Equal(t, "test-partition", req.Tenancy.Partition)
+
+						require.Equal(t, discovery.LookupTypeService, reqType)
+					})
+			},
+			validateAndNormalizeExpected: true,
+			response: &dns.Msg{
+				MsgHdr: dns.MsgHdr{
+					Response:      true,
+					Authoritative: true,
+				},
+				Compress: true,
+				Question: []dns.Question{
+					{
+						Name:   "foo.service.consul.",
+						Qtype:  dns.TypeA,
+						Qclass: dns.ClassINET,
+					},
+				},
+				Answer: []dns.RR{
+					&dns.A{
+						Hdr: dns.RR_Header{
+							Name:   "foo.service.consul.",
+							Rrtype: dns.TypeA,
+							Class:  dns.ClassINET,
+							Ttl:    123,
+						},
+						A: net.ParseIP("1.2.3.4"),
+					},
+				},
+			},
+		},
+		{
+			name: "When a request context is provided, values do not override explicit tenancy",
+			request: &dns.Msg{
+				MsgHdr: dns.MsgHdr{
+					Opcode: dns.OpcodeQuery,
+				},
+				Question: []dns.Question{
+					{
+						Name:   "foo.service.bar.ns.baz.ap.consul.",
+						Qtype:  dns.TypeA,
+						Qclass: dns.ClassINET,
+					},
+				},
+			},
+			requestContext: &Context{
+				Token:            "test-token",
+				DefaultNamespace: "test-namespace",
+				DefaultPartition: "test-partition",
+			},
+			configureDataFetcher: func(fetcher discovery.CatalogDataFetcher) {
+				result := []*discovery.Result{
+					{
+						Type: discovery.ResultTypeNode,
+						Node: &discovery.Location{Name: "foo", Address: "1.2.3.4"},
+						Tenancy: discovery.ResultTenancy{
+							Namespace: "bar",
+							Partition: "baz",
+						},
+					},
+				}
+
+				fetcher.(*discovery.MockCatalogDataFetcher).
+					On("FetchEndpoints", mock.Anything, mock.Anything, mock.Anything).
+					Return(result, nil).
+					Run(func(args mock.Arguments) {
+						ctx := args.Get(0).(discovery.Context)
+						req := args.Get(1).(*discovery.QueryPayload)
+						reqType := args.Get(2).(discovery.LookupType)
+
+						require.Equal(t, "test-token", ctx.Token)
+
+						require.Equal(t, "foo", req.Name)
+						require.Equal(t, "bar", req.Tenancy.Namespace)
+						require.Equal(t, "baz", req.Tenancy.Partition)
+
+						require.Equal(t, discovery.LookupTypeService, reqType)
+					})
+			},
+			validateAndNormalizeExpected: true,
+			response: &dns.Msg{
+				MsgHdr: dns.MsgHdr{
+					Response:      true,
+					Authoritative: true,
+				},
+				Compress: true,
+				Question: []dns.Question{
+					{
+						Name:   "foo.service.bar.ns.baz.ap.consul.",
+						Qtype:  dns.TypeA,
+						Qclass: dns.ClassINET,
+					},
+				},
+				Answer: []dns.RR{
+					&dns.A{
+						Hdr: dns.RR_Header{
+							Name:   "foo.service.bar.ns.baz.ap.consul.",
+							Rrtype: dns.TypeA,
+							Class:  dns.ClassINET,
+							Ttl:    123,
+						},
+						A: net.ParseIP("1.2.3.4"),
+					},
+				},
+			},
+		},
 	}
 
 	for _, tc := range testCases {

agent/grpc-external/services/dns/server_v2.go

@@ -72,9 +72,10 @@ func (s *ServerV2) Query(ctx context.Context, req *pbdns.QueryRequest) (*pbdns.Q
 		return nil, status.Error(codes.Internal, fmt.Sprintf("failure decoding dns request: %s", err.Error()))
 	}
 
-	// TODO (v2-dns): parse token and other context metadata from the grpc request/metadata (NET-7885)
-	reqCtx := agentdns.Context{
-		Token: s.TokenFunc(),
+	reqCtx, err := agentdns.NewContextFromGRPCContext(ctx)
+	if err != nil {
+		s.Logger.Error("error parsing DNS context from grpc metadata", "err", err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("error parsing DNS context from grpc metadata: %s", err.Error()))
 	}
 
 	resp := s.DNSRouter.HandleRequest(msg, reqCtx, remote)

agent/grpc-external/services/dns/server_v2_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/hashicorp/go-hclog"
 	"github.com/miekg/dns"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/metadata"
 
 	agentdns "github.com/hashicorp/consul/agent/dns"
 	"github.com/hashicorp/consul/proto-public/pbdns"
@@ -50,6 +52,7 @@ func (s *DNSTestSuite) TestProxy_V2Success() {
 		question        string
 		configureRouter func(router *agentdns.MockDNSRouter)
 		clientQuery     func(qR *pbdns.QueryRequest)
+		metadata        map[string]string
 		expectedErr     error
 	}{
 
@@ -73,6 +76,28 @@ func (s *DNSTestSuite) TestProxy_V2Success() {
 				qR.Protocol = pbdns.Protocol_PROTOCOL_TCP
 			},
 		},
+		"happy path with context variables set": {
+			question: "abc.com.",
+			configureRouter: func(router *agentdns.MockDNSRouter) {
+				router.On("HandleRequest", mock.Anything, mock.Anything, mock.Anything).
+					Run(func(args mock.Arguments) {
+						ctx, ok := args.Get(1).(agentdns.Context)
+						require.True(s.T(), ok, "error casting to agentdns.Context")
+						require.Equal(s.T(), "test-token", ctx.Token, "token not set in context")
+						require.Equal(s.T(), "test-namespace", ctx.DefaultNamespace, "namespace not set in context")
+						require.Equal(s.T(), "test-partition", ctx.DefaultPartition, "partition not set in context")
+					}).
+					Return(basicResponse(), nil)
+			},
+			clientQuery: func(qR *pbdns.QueryRequest) {
+				qR.Protocol = pbdns.Protocol_PROTOCOL_UDP
+			},
+			metadata: map[string]string{
+				"x-consul-token":     "test-token",
+				"x-consul-namespace": "test-namespace",
+				"x-consul-partition": "test-partition",
+			},
+		},
 		"No protocol set": {
 			question:    "abc.com.",
 			clientQuery: func(qR *pbdns.QueryRequest) {},
@@ -108,9 +133,18 @@ func (s *DNSTestSuite) TestProxy_V2Success() {
 
 			bytes, _ := req.Pack()
 
+			ctx := context.Background()
+			if len(tc.metadata) > 0 {
+				md := metadata.MD{}
+				for k, v := range tc.metadata {
+					md.Set(k, v)
+				}
+				ctx = metadata.NewOutgoingContext(ctx, md)
+			}
+
 			clientReq := &pbdns.QueryRequest{Msg: bytes}
 			tc.clientQuery(clientReq)
-			clientResp, err := client.Query(context.Background(), clientReq)
+			clientResp, err := client.Query(ctx, clientReq)
 			if tc.expectedErr != nil {
 				s.Require().Error(err, "no errror calling gRPC endpoint")
 				s.Require().ErrorContains(err, tc.expectedErr.Error())

agent/proxycfg-glue/config_entry.go

@@ -69,6 +69,8 @@ func newConfigEntryRequest(req *structs.ConfigEntryQuery, deps ServerDataSourceD
 		topic = pbsubscribe.Topic_HTTPRoute
 	case structs.TCPRoute:
 		topic = pbsubscribe.Topic_TCPRoute
+	case structs.FileSystemCertificate:
+		topic = pbsubscribe.Topic_FileSystemCertificate
 	case structs.InlineCertificate:
 		topic = pbsubscribe.Topic_InlineCertificate
 	case structs.BoundAPIGateway:

agent/proxycfg/api_gateway.go

@@ -63,7 +63,8 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err
 	snap.APIGateway.BoundListeners = make(map[string]structs.BoundAPIGatewayListener)
 	snap.APIGateway.HTTPRoutes = watch.NewMap[structs.ResourceReference, *structs.HTTPRouteConfigEntry]()
 	snap.APIGateway.TCPRoutes = watch.NewMap[structs.ResourceReference, *structs.TCPRouteConfigEntry]()
-	snap.APIGateway.Certificates = watch.NewMap[structs.ResourceReference, *structs.InlineCertificateConfigEntry]()
+	snap.APIGateway.InlineCertificates = watch.NewMap[structs.ResourceReference, *structs.InlineCertificateConfigEntry]()
+	snap.APIGateway.FileSystemCertificates = watch.NewMap[structs.ResourceReference, *structs.FileSystemCertificateConfigEntry]()
 
 	snap.APIGateway.Upstreams = make(listenerRouteUpstreams)
 	snap.APIGateway.UpstreamsSet = make(routeUpstreamSet)
@@ -96,7 +97,8 @@ func (h *handlerAPIGateway) subscribeToConfigEntry(ctx context.Context, kind, na
 // handleUpdate responds to changes in the api-gateway. In general, we want
 // to crawl the various resources related to or attached to the gateway and
 // collect the list of things need to generate xDS.  This list of resources
-// includes the bound-api-gateway, http-routes, tcp-routes, and inline-certificates.
+// includes the bound-api-gateway, http-routes, tcp-routes,
+// file-system-certificates and inline-certificates.
 func (h *handlerAPIGateway) handleUpdate(ctx context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
 	if u.Err != nil {
 		return fmt.Errorf("error filling agent cache: %v", u.Err)
@@ -113,6 +115,11 @@ func (h *handlerAPIGateway) handleUpdate(ctx context.Context, u UpdateEvent, sna
 		if err := h.handleGatewayConfigUpdate(ctx, u, snap, u.CorrelationID); err != nil {
 			return err
 		}
+	case fileSystemCertificateConfigWatchID:
+		// Handle change in an attached file-system-certificate config entry
+		if err := h.handleFileSystemCertConfigUpdate(ctx, u, snap); err != nil {
+			return err
+		}
 	case inlineCertificateConfigWatchID:
 		// Handle change in an attached inline-certificate config entry
 		if err := h.handleInlineCertConfigUpdate(ctx, u, snap); err != nil {
@@ -205,12 +212,21 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
 			for _, ref := range listener.Certificates {
 				ctx, cancel := context.WithCancel(ctx)
 				seenRefs[ref] = struct{}{}
-				snap.APIGateway.Certificates.InitWatch(ref, cancel)
 
-				err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, inlineCertificateConfigWatchID)
-				if err != nil {
-					// TODO May want to continue
-					return err
+				if ref.Kind == structs.FileSystemCertificate {
+					snap.APIGateway.FileSystemCertificates.InitWatch(ref, cancel)
+
+					err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, fileSystemCertificateConfigWatchID)
+					if err != nil {
+						return err
+					}
+				} else {
+					snap.APIGateway.InlineCertificates.InitWatch(ref, cancel)
+
+					err := h.subscribeToConfigEntry(ctx, ref.Kind, ref.Name, ref.EnterpriseMeta, inlineCertificateConfigWatchID)
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}
@@ -234,9 +250,16 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
 			return true
 		})
 
-		snap.APIGateway.Certificates.ForEachKey(func(ref structs.ResourceReference) bool {
+		snap.APIGateway.InlineCertificates.ForEachKey(func(ref structs.ResourceReference) bool {
 			if _, ok := seenRefs[ref]; !ok {
-				snap.APIGateway.Certificates.CancelWatch(ref)
+				snap.APIGateway.InlineCertificates.CancelWatch(ref)
+			}
+			return true
+		})
+
+		snap.APIGateway.FileSystemCertificates.ForEachKey(func(ref structs.ResourceReference) bool {
+			if _, ok := seenRefs[ref]; !ok {
+				snap.APIGateway.FileSystemCertificates.CancelWatch(ref)
 			}
 			return true
 		})
@@ -265,6 +288,30 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
 	return h.watchIngressLeafCert(ctx, snap)
 }
 
+func (h *handlerAPIGateway) handleFileSystemCertConfigUpdate(_ context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
+	resp, ok := u.Result.(*structs.ConfigEntryResponse)
+	if !ok {
+		return fmt.Errorf("invalid type for response: %T", u.Result)
+	} else if resp == nil || resp.Entry == nil {
+		return nil
+	}
+
+	cfg, ok := resp.Entry.(*structs.FileSystemCertificateConfigEntry)
+	if !ok || cfg == nil {
+		return fmt.Errorf("invalid type for config entry: %T", resp.Entry)
+	}
+
+	ref := structs.ResourceReference{
+		Kind:           cfg.GetKind(),
+		Name:           cfg.GetName(),
+		EnterpriseMeta: *cfg.GetEnterpriseMeta(),
+	}
+
+	snap.APIGateway.FileSystemCertificates.Set(ref, cfg)
+
+	return nil
+}
+
 // handleInlineCertConfigUpdate stores the certificate for the gateway
 func (h *handlerAPIGateway) handleInlineCertConfigUpdate(_ context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
 	resp, ok := u.Result.(*structs.ConfigEntryResponse)
@@ -285,7 +332,7 @@ func (h *handlerAPIGateway) handleInlineCertConfigUpdate(_ context.Context, u Up
 		EnterpriseMeta: *cfg.GetEnterpriseMeta(),
 	}
 
-	snap.APIGateway.Certificates.Set(ref, cfg)
+	snap.APIGateway.InlineCertificates.Set(ref, cfg)
 
 	return nil
 }

agent/proxycfg/mesh_gateway.go

@@ -127,6 +127,17 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er
 	if err != nil {
 		return snap, err
 	}
+	// Watch for service default object that matches this mesh gateway's name
+	err = s.dataSources.ConfigEntry.Notify(ctx, &structs.ConfigEntryQuery{
+		Kind:           structs.ServiceDefaults,
+		Name:           s.service,
+		Datacenter:     s.source.Datacenter,
+		QueryOptions:   structs.QueryOptions{Token: s.token},
+		EnterpriseMeta: s.proxyID.EnterpriseMeta,
+	}, serviceDefaultsWatchID, s.ch)
+	if err != nil {
+		return snap, err
+	}
 
 	snap.MeshGateway.WatchedServices = make(map[structs.ServiceName]context.CancelFunc)
 	snap.MeshGateway.WatchedGateways = make(map[string]context.CancelFunc)
@@ -648,6 +659,25 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u UpdateEvent, sn
 		}
 
 		snap.MeshGateway.PeerServers = peerServers
+	case serviceDefaultsWatchID:
+		resp, ok := u.Result.(*structs.ConfigEntryResponse)
+		if !ok {
+			return fmt.Errorf("invalid type for config entry: %T", resp.Entry)
+		}
+
+		if resp.Entry == nil {
+			return nil
+		}
+		serviceDefaults, ok := resp.Entry.(*structs.ServiceConfigEntry)
+		if !ok {
+			return fmt.Errorf("invalid type for config entry: %T", resp.Entry)
+		}
+
+		if serviceDefaults.UpstreamConfig != nil && serviceDefaults.UpstreamConfig.Defaults != nil {
+			if serviceDefaults.UpstreamConfig.Defaults.Limits != nil {
+				snap.MeshGateway.Limits = serviceDefaults.UpstreamConfig.Defaults.Limits
+			}
+		}
 
 	default:
 		switch {

agent/proxycfg/proxycfg.deepcopy.go

@@ -421,7 +421,8 @@ func (o *configSnapshotAPIGateway) DeepCopy() *configSnapshotAPIGateway {
 	}
 	cp.HTTPRoutes = o.HTTPRoutes.DeepCopy()
 	cp.TCPRoutes = o.TCPRoutes.DeepCopy()
-	cp.Certificates = o.Certificates.DeepCopy()
+	cp.InlineCertificates = o.InlineCertificates.DeepCopy()
+	cp.FileSystemCertificates = o.FileSystemCertificates.DeepCopy()
 	if o.Listeners != nil {
 		cp.Listeners = make(map[string]structs.APIGatewayListener, len(o.Listeners))
 		for k2, v2 := range o.Listeners {
@@ -727,6 +728,22 @@ func (o *configSnapshotMeshGateway) DeepCopy() *configSnapshotMeshGateway {
 			}
 		}
 	}
+	if o.Limits != nil {
+		cp.Limits = new(structs.UpstreamLimits)
+		*cp.Limits = *o.Limits
+		if o.Limits.MaxConnections != nil {
+			cp.Limits.MaxConnections = new(int)
+			*cp.Limits.MaxConnections = *o.Limits.MaxConnections
+		}
+		if o.Limits.MaxPendingRequests != nil {
+			cp.Limits.MaxPendingRequests = new(int)
+			*cp.Limits.MaxPendingRequests = *o.Limits.MaxPendingRequests
+		}
+		if o.Limits.MaxConcurrentRequests != nil {
+			cp.Limits.MaxConcurrentRequests = new(int)
+			*cp.Limits.MaxConcurrentRequests = *o.Limits.MaxConcurrentRequests
+		}
+	}
 	return &cp
 }
 

agent/proxycfg/snapshot.go

@@ -9,6 +9,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/hashicorp/go-hclog"
+
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/agent/consul/discoverychain"
@@ -17,7 +19,6 @@ import (
 	"github.com/hashicorp/consul/agent/xds/config"
 	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/proto/private/pbpeering"
-	"github.com/hashicorp/go-hclog"
 )
 
 // TODO(ingress): Can we think of a better for this bag of data?
@@ -497,6 +498,9 @@ type configSnapshotMeshGateway struct {
 	// PeeringTrustBundlesSet indicates that the watch on the peer trust
 	// bundles has completed at least once.
 	PeeringTrustBundlesSet bool
+
+	// Limits
+	Limits *structs.UpstreamLimits
 }
 
 // MeshGatewayValidExportedServices ensures that the following data is present
@@ -735,9 +739,11 @@ type configSnapshotAPIGateway struct {
 	// UpstreamsSet is the unique set of UpstreamID the gateway routes to.
 	UpstreamsSet routeUpstreamSet
 
-	HTTPRoutes   watch.Map[structs.ResourceReference, *structs.HTTPRouteConfigEntry]
-	TCPRoutes    watch.Map[structs.ResourceReference, *structs.TCPRouteConfigEntry]
-	Certificates watch.Map[structs.ResourceReference, *structs.InlineCertificateConfigEntry]
+	HTTPRoutes watch.Map[structs.ResourceReference, *structs.HTTPRouteConfigEntry]
+	TCPRoutes  watch.Map[structs.ResourceReference, *structs.TCPRouteConfigEntry]
+
+	InlineCertificates     watch.Map[structs.ResourceReference, *structs.InlineCertificateConfigEntry]
+	FileSystemCertificates watch.Map[structs.ResourceReference, *structs.FileSystemCertificateConfigEntry]
 
 	// LeafCertWatchCancel is a CancelFunc to use when refreshing this gateway's
 	// leaf cert watch with different parameters.

agent/proxycfg/state.go

@@ -34,10 +34,12 @@ const (
 	consulServerListWatchID            = "consul-server-list"
 	datacentersWatchID                 = "datacenters"
 	serviceResolversWatchID            = "service-resolvers"
+	serviceDefaultsWatchID             = "service-defaults"
 	gatewayServicesWatchID             = "gateway-services"
 	gatewayConfigWatchID               = "gateway-config"
 	apiGatewayConfigWatchID            = "api-gateway-config"
 	boundGatewayConfigWatchID          = "bound-gateway-config"
+	fileSystemCertificateConfigWatchID = "file-system-certificate-config"
 	inlineCertificateConfigWatchID     = "inline-certificate-config"
 	routeConfigWatchID                 = "route-config"
 	externalServiceIDPrefix            = "external-service:"

agent/proxycfg/testing_mesh_gateway.go

@@ -264,6 +264,25 @@ func TestConfigSnapshotMeshGateway(t testing.T, variant string, nsFn func(ns *st
 				},
 			},
 		})
+	case "limits-added":
+		extraUpdates = append(extraUpdates, UpdateEvent{
+			CorrelationID: serviceDefaultsWatchID,
+			Result: &structs.ConfigEntryResponse{
+				Entry: &structs.ServiceConfigEntry{
+					Kind: structs.ServiceDefaults,
+					Name: "mesh-gateway",
+					UpstreamConfig: &structs.UpstreamConfiguration{
+						Defaults: &structs.UpstreamConfig{
+							Limits: &structs.UpstreamLimits{
+								MaxConnections:        pointerTo(1),
+								MaxPendingRequests:    pointerTo(10),
+								MaxConcurrentRequests: pointerTo(100),
+							},
+						},
+					},
+				},
+			},
+		})
 	default:
 		t.Fatalf("unknown variant: %s", variant)
 		return nil
@@ -1124,3 +1143,7 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func(
 		},
 	}, nsFn, nil, testSpliceEvents(baseEvents, extraUpdates))
 }
+
+func pointerTo[T any](v T) *T {
+	return &v
+}

agent/structs/config_entry.go

@@ -27,22 +27,23 @@ import (
 )
 
 const (
-	ServiceDefaults    string = "service-defaults"
-	ProxyDefaults      string = "proxy-defaults"
-	ServiceRouter      string = "service-router"
-	ServiceSplitter    string = "service-splitter"
-	ServiceResolver    string = "service-resolver"
-	IngressGateway     string = "ingress-gateway"
-	TerminatingGateway string = "terminating-gateway"
-	ServiceIntentions  string = "service-intentions"
-	MeshConfig         string = "mesh"
-	ExportedServices   string = "exported-services"
-	SamenessGroup      string = "sameness-group"
-	APIGateway         string = "api-gateway"
-	BoundAPIGateway    string = "bound-api-gateway"
-	InlineCertificate  string = "inline-certificate"
-	HTTPRoute          string = "http-route"
-	TCPRoute           string = "tcp-route"
+	ServiceDefaults       string = "service-defaults"
+	ProxyDefaults         string = "proxy-defaults"
+	ServiceRouter         string = "service-router"
+	ServiceSplitter       string = "service-splitter"
+	ServiceResolver       string = "service-resolver"
+	IngressGateway        string = "ingress-gateway"
+	TerminatingGateway    string = "terminating-gateway"
+	ServiceIntentions     string = "service-intentions"
+	MeshConfig            string = "mesh"
+	ExportedServices      string = "exported-services"
+	SamenessGroup         string = "sameness-group"
+	APIGateway            string = "api-gateway"
+	BoundAPIGateway       string = "bound-api-gateway"
+	FileSystemCertificate string = "file-system-certificate"
+	InlineCertificate     string = "inline-certificate"
+	HTTPRoute             string = "http-route"
+	TCPRoute              string = "tcp-route"
 	// TODO: decide if we want to highlight 'ip' keyword in the name of RateLimitIPConfig
 	RateLimitIPConfig string = "control-plane-request-limit"
 	JWTProvider       string = "jwt-provider"
@@ -71,6 +72,7 @@ var AllConfigEntryKinds = []string{
 	BoundAPIGateway,
 	HTTPRoute,
 	TCPRoute,
+	FileSystemCertificate,
 	InlineCertificate,
 	RateLimitIPConfig,
 	JWTProvider,
@@ -832,6 +834,8 @@ func MakeConfigEntry(kind, name string) (ConfigEntry, error) {
 		return &APIGatewayConfigEntry{Name: name}, nil
 	case BoundAPIGateway:
 		return &BoundAPIGatewayConfigEntry{Name: name}, nil
+	case FileSystemCertificate:
+		return &FileSystemCertificateConfigEntry{Name: name}, nil
 	case InlineCertificate:
 		return &InlineCertificateConfigEntry{Name: name}, nil
 	case HTTPRoute:

agent/structs/config_entry_file_system_certificate.go

@@ -0,0 +1,73 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package structs
+
+import (
+	"github.com/hashicorp/consul/acl"
+)
+
+// FileSystemCertificateConfigEntry manages the configuration for a certificate
+// and private key located in the local file system.
+type FileSystemCertificateConfigEntry struct {
+	// Kind of config entry. This will be set to structs.FileSystemCertificate.
+	Kind string
+
+	// Name is used to match the config entry with its associated file system certificate.
+	Name string
+
+	// Certificate is the optional path to a client certificate to use for TLS connections.
+	Certificate string
+
+	// PrivateKey is the optional path to a private key to use for TLS connections.
+	PrivateKey string
+
+	Meta               map[string]string `json:",omitempty"`
+	Hash               uint64            `json:",omitempty" hash:"ignore"`
+	acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"`
+	RaftIndex          `hash:"ignore"`
+}
+
+func (e *FileSystemCertificateConfigEntry) SetHash(h uint64) {
+	e.Hash = h
+}
+
+func (e *FileSystemCertificateConfigEntry) GetHash() uint64 {
+	return e.Hash
+}
+
+func (e *FileSystemCertificateConfigEntry) GetKind() string { return FileSystemCertificate }
+func (e *FileSystemCertificateConfigEntry) GetName() string { return e.Name }
+func (e *FileSystemCertificateConfigEntry) Normalize() error {
+	h, err := HashConfigEntry(e)
+	if err != nil {
+		return err
+	}
+	e.Hash = h
+	return nil
+}
+func (e *FileSystemCertificateConfigEntry) GetMeta() map[string]string { return e.Meta }
+func (e *FileSystemCertificateConfigEntry) GetEnterpriseMeta() *acl.EnterpriseMeta {
+	return &e.EnterpriseMeta
+}
+func (e *FileSystemCertificateConfigEntry) GetRaftIndex() *RaftIndex { return &e.RaftIndex }
+
+func (e *FileSystemCertificateConfigEntry) Validate() error {
+	return validateConfigEntryMeta(e.Meta)
+}
+
+func (e *FileSystemCertificateConfigEntry) Hosts() ([]string, error) {
+	return []string{}, nil
+}
+
+func (e *FileSystemCertificateConfigEntry) CanRead(authz acl.Authorizer) error {
+	var authzContext acl.AuthorizerContext
+	e.FillAuthzContext(&authzContext)
+	return authz.ToAllowAuthorizer().MeshReadAllowed(&authzContext)
+}
+
+func (e *FileSystemCertificateConfigEntry) CanWrite(authz acl.Authorizer) error {
+	var authzContext acl.AuthorizerContext
+	e.FillAuthzContext(&authzContext)
+	return authz.ToAllowAuthorizer().MeshWriteAllowed(&authzContext)
+}

agent/structs/config_entry_gateways.go

@@ -870,7 +870,8 @@ func (e *APIGatewayConfigEntry) validateListeners() error {
 		ListenerProtocolTCP:  true,
 	}
 	allowedCertificateKinds := map[string]bool{
-		InlineCertificate: true,
+		FileSystemCertificate: true,
+		InlineCertificate:     true,
 	}
 
 	for _, listener := range e.Listeners {
@@ -889,7 +890,7 @@ func (e *APIGatewayConfigEntry) validateListeners() error {
 		}
 		for _, certificate := range listener.TLS.Certificates {
 			if !allowedCertificateKinds[certificate.Kind] {
-				return fmt.Errorf("unsupported certificate kind: %q, must be 'inline-certificate'", certificate.Kind)
+				return fmt.Errorf("unsupported certificate kind: %q, must be 'file-system-certificate' or 'inline-certificate'", certificate.Kind)
 			}
 			if certificate.Name == "" {
 				return fmt.Errorf("certificate reference must have a name")
@@ -978,6 +979,16 @@ func (a *APIGatewayTLSConfiguration) IsEmpty() bool {
 	return len(a.Certificates) == 0 && len(a.MaxVersion) == 0 && len(a.MinVersion) == 0 && len(a.CipherSuites) == 0
 }
 
+func (a *APIGatewayTLSConfiguration) ToGatewayTLSConfig() GatewayTLSConfig {
+	return GatewayTLSConfig{
+		Enabled:       true,
+		SDS:           nil,
+		TLSMinVersion: a.MinVersion,
+		TLSMaxVersion: a.MaxVersion,
+		CipherSuites:  a.CipherSuites,
+	}
+}
+
 // ServiceRouteReferences is a map with a key of ServiceName type for a routed to service from a
 // bound gateway listener with a value being a slice of resource references of the routes that reference the service
 type ServiceRouteReferences map[ServiceName][]ResourceReference

agent/structs/deep-copy.sh

@@ -24,6 +24,7 @@ deep-copy \
   -type DiscoverySplit \
   -type ExposeConfig \
   -type ExportedServicesConfigEntry \
+  -type FileSystemCertificateConfigEntry \
   -type GatewayService \
   -type GatewayServiceTLSConfig \
   -type HTTPHeaderModifiers \

agent/structs/structs.deepcopy.go

@@ -1,4 +1,4 @@
-// generated by deep-copy -pointer-receiver -o ./structs.deepcopy.go -type APIGatewayListener -type BoundAPIGatewayListener -type CARoot -type CheckServiceNode -type CheckType -type CompiledDiscoveryChain -type ConnectProxyConfig -type DiscoveryFailover -type DiscoveryGraphNode -type DiscoveryResolver -type DiscoveryRoute -type DiscoverySplit -type ExposeConfig -type ExportedServicesConfigEntry -type GatewayService -type GatewayServiceTLSConfig -type HTTPHeaderModifiers -type HTTPRouteConfigEntry -type HashPolicy -type HealthCheck -type IndexedCARoots -type IngressListener -type InlineCertificateConfigEntry -type Intention -type IntentionPermission -type LoadBalancer -type MeshConfigEntry -type MeshDirectionalTLSConfig -type MeshTLSConfig -type Node -type NodeService -type PeeringServiceMeta -type ServiceConfigEntry -type ServiceConfigResponse -type ServiceConnect -type ServiceDefinition -type ServiceResolverConfigEntry -type ServiceResolverFailover -type ServiceRoute -type ServiceRouteDestination -type ServiceRouteMatch -type TCPRouteConfigEntry -type Upstream -type UpstreamConfiguration -type Status -type BoundAPIGatewayConfigEntry ./; DO NOT EDIT.
+// generated by deep-copy -pointer-receiver -o ./structs.deepcopy.go -type APIGatewayListener -type BoundAPIGatewayListener -type CARoot -type CheckServiceNode -type CheckType -type CompiledDiscoveryChain -type ConnectProxyConfig -type DiscoveryFailover -type DiscoveryGraphNode -type DiscoveryResolver -type DiscoveryRoute -type DiscoverySplit -type ExposeConfig -type ExportedServicesConfigEntry -type FileSystemCertificateConfigEntry -type GatewayService -type GatewayServiceTLSConfig -type HTTPHeaderModifiers -type HTTPRouteConfigEntry -type HashPolicy -type HealthCheck -type IndexedCARoots -type IngressListener -type InlineCertificateConfigEntry -type Intention -type IntentionPermission -type LoadBalancer -type MeshConfigEntry -type MeshDirectionalTLSConfig -type MeshTLSConfig -type Node -type NodeService -type PeeringServiceMeta -type ServiceConfigEntry -type ServiceConfigResponse -type ServiceConnect -type ServiceDefinition -type ServiceResolverConfigEntry -type ServiceResolverFailover -type ServiceRoute -type ServiceRouteDestination -type ServiceRouteMatch -type TCPRouteConfigEntry -type Upstream -type UpstreamConfiguration -type Status -type BoundAPIGatewayConfigEntry ./; DO NOT EDIT.
 
 package structs
 
@@ -318,6 +318,18 @@ func (o *ExportedServicesConfigEntry) DeepCopy() *ExportedServicesConfigEntry {
 	return &cp
 }
 
+// DeepCopy generates a deep copy of *FileSystemCertificateConfigEntry
+func (o *FileSystemCertificateConfigEntry) DeepCopy() *FileSystemCertificateConfigEntry {
+	var cp FileSystemCertificateConfigEntry = *o
+	if o.Meta != nil {
+		cp.Meta = make(map[string]string, len(o.Meta))
+		for k2, v2 := range o.Meta {
+			cp.Meta[k2] = v2
+		}
+	}
+	return &cp
+}
+
 // DeepCopy generates a deep copy of *GatewayService
 func (o *GatewayService) DeepCopy() *GatewayService {
 	var cp GatewayService = *o

agent/xds/clusters.go

@@ -532,6 +532,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
 			name:              connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain),
 			hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[key.String()],
 			isRemote:          true,
+			limits:            cfgSnap.MeshGateway.Limits,
 		}
 		cluster := s.makeGatewayCluster(cfgSnap, opts)
 		clusters = append(clusters, cluster)
@@ -554,6 +555,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
 				name:              cfgSnap.ServerSNIFn(key.Datacenter, ""),
 				hostnameEndpoints: hostnameEndpoints,
 				isRemote:          !key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrDefault()),
+				limits:            cfgSnap.MeshGateway.Limits,
 			}
 			cluster := s.makeGatewayCluster(cfgSnap, opts)
 			clusters = append(clusters, cluster)
@@ -563,7 +565,8 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
 		servers, _ := cfgSnap.MeshGateway.WatchedLocalServers.Get(structs.ConsulServiceName)
 		for _, srv := range servers {
 			opts := clusterOpts{
-				name: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node),
+				name:   cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node),
+				limits: cfgSnap.MeshGateway.Limits,
 			}
 			cluster := s.makeGatewayCluster(cfgSnap, opts)
 			clusters = append(clusters, cluster)
@@ -579,14 +582,15 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
 		// We avoid routing to read replicas since they will never be Raft voters.
 		if haveVoters(servers) {
 			cluster := s.makeGatewayCluster(cfgSnap, clusterOpts{
-				name: connect.PeeringServerSAN(cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain),
+				name:   connect.PeeringServerSAN(cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain),
+				limits: cfgSnap.MeshGateway.Limits,
 			})
 			clusters = append(clusters, cluster)
 		}
 	}
 
 	// generate the per-service/subset clusters
-	c, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.MeshGateway.ServiceGroups, cfgSnap.MeshGateway.ServiceResolvers)
+	c, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.MeshGateway.ServiceGroups, cfgSnap.MeshGateway.ServiceResolvers, cfgSnap.MeshGateway.Limits)
 	if err != nil {
 		return nil, err
 	}
@@ -664,7 +668,7 @@ func (s *ResourceGenerator) makePeerServerClusters(cfgSnap *proxycfg.ConfigSnaps
 // for a terminating gateway. This will include 1 cluster per Destination associated with this terminating gateway.
 func (s *ResourceGenerator) clustersFromSnapshotTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
 	res := []proto.Message{}
-	gwClusters, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers)
+	gwClusters, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -683,6 +687,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
 	cfgSnap *proxycfg.ConfigSnapshot,
 	services map[structs.ServiceName]structs.CheckServiceNodes,
 	resolvers map[structs.ServiceName]*structs.ServiceResolverConfigEntry,
+	limits *structs.UpstreamLimits,
 ) ([]proto.Message, error) {
 	var hostnameEndpoints structs.CheckServiceNodes
 
@@ -724,6 +729,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
 			hostnameEndpoints: hostnameEndpoints,
 			connectTimeout:    resolver.ConnectTimeout,
 			isRemote:          isRemote,
+			limits:            limits,
 		}
 		cluster := s.makeGatewayCluster(cfgSnap, opts)
 
@@ -763,6 +769,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
 				onlyPassing:       subset.OnlyPassing,
 				connectTimeout:    resolver.ConnectTimeout,
 				isRemote:          isRemote,
+				limits:            limits,
 			}
 			cluster := s.makeGatewayCluster(cfgSnap, opts)
 
@@ -812,6 +819,7 @@ func (s *ResourceGenerator) makeGatewayOutgoingClusterPeeringServiceClusters(cfg
 				name:              clusterName,
 				isRemote:          true,
 				hostnameEndpoints: hostnameEndpoints,
+				limits:            cfgSnap.MeshGateway.Limits,
 			}
 			cluster := s.makeGatewayCluster(cfgSnap, opts)
 
@@ -1706,6 +1714,8 @@ type clusterOpts struct {
 	// Corresponds to a valid address/port pairs to be routed externally
 	// these addresses will be embedded in the cluster configuration and will never use EDS
 	addresses []structs.ServiceAddress
+
+	limits *structs.UpstreamLimits
 }
 
 // makeGatewayCluster creates an Envoy cluster for a mesh or terminating gateway
@@ -1768,6 +1778,12 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op
 		)
 	}
 
+	if opts.limits != nil {
+		cluster.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{
+			Thresholds: makeThresholdsIfNeeded(opts.limits),
+		}
+	}
+
 	return cluster
 }
 

agent/xds/listeners_apigateway.go

@@ -6,22 +6,19 @@ package xds
 import (
 	"fmt"
 
-	"golang.org/x/exp/maps"
-
 	envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	envoy_http_jwt_authn_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/jwt_authn/v3"
 	envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-
-	"github.com/hashicorp/consul/agent/consul/discoverychain"
-	"github.com/hashicorp/consul/agent/xds/naming"
-
+	"golang.org/x/exp/maps"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
+	"github.com/hashicorp/consul/agent/consul/discoverychain"
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/agent/xds/naming"
 	"github.com/hashicorp/consul/types"
 )
 
@@ -35,13 +32,19 @@ func (s *ResourceGenerator) makeAPIGatewayListeners(address string, cfgSnap *pro
 		listenerKey := readyListener.listenerKey
 		boundListener := readyListener.boundListenerCfg
 
-		var certs []structs.InlineCertificateConfigEntry
+		// Collect the referenced certificate config entries
+		var certs []structs.ConfigEntry
 		for _, certRef := range boundListener.Certificates {
-			cert, ok := cfgSnap.APIGateway.Certificates.Get(certRef)
-			if !ok {
-				continue
+			switch certRef.Kind {
+			case structs.InlineCertificate:
+				if cert, ok := cfgSnap.APIGateway.InlineCertificates.Get(certRef); ok {
+					certs = append(certs, cert)
+				}
+			case structs.FileSystemCertificate:
+				if cert, ok := cfgSnap.APIGateway.FileSystemCertificates.Get(certRef); ok {
+					certs = append(certs, cert)
+				}
 			}
-			certs = append(certs, *cert)
 		}
 
 		isAPIGatewayWithTLS := len(boundListener.Certificates) > 0
@@ -111,7 +114,7 @@ func (s *ResourceGenerator) makeAPIGatewayListeners(address string, cfgSnap *pro
 
 			if isAPIGatewayWithTLS {
 				// construct SNI filter chains
-				l.FilterChains, err = makeInlineOverrideFilterChains(
+				l.FilterChains, err = s.makeInlineOverrideFilterChains(
 					cfgSnap,
 					cfgSnap.APIGateway.TLSConfig,
 					listenerKey.Protocol,
@@ -225,7 +228,7 @@ func (s *ResourceGenerator) makeAPIGatewayListeners(address string, cfgSnap *pro
 			sniFilterChains := []*envoy_listener_v3.FilterChain{}
 
 			if isAPIGatewayWithTLS {
-				sniFilterChains, err = makeInlineOverrideFilterChains(cfgSnap, cfgSnap.IngressGateway.TLSConfig, listenerKey.Protocol, filterOpts, certs)
+				sniFilterChains, err = s.makeInlineOverrideFilterChains(cfgSnap, cfgSnap.IngressGateway.TLSConfig, listenerKey.Protocol, filterOpts, certs)
 				if err != nil {
 					return nil, err
 				}
@@ -393,11 +396,11 @@ func resolveAPIListenerTLSConfig(listenerTLSCfg structs.APIGatewayTLSConfigurati
 
 // when we have multiple certificates on a single listener, we need
 // to duplicate the filter chains with multiple TLS contexts
-func makeInlineOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot,
+func (s *ResourceGenerator) makeInlineOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot,
 	tlsCfg structs.GatewayTLSConfig,
 	protocol string,
 	filterOpts listenerFilterOpts,
-	certs []structs.InlineCertificateConfigEntry,
+	certs []structs.ConfigEntry,
 ) ([]*envoy_listener_v3.FilterChain, error) {
 	var chains []*envoy_listener_v3.FilterChain
 
@@ -438,50 +441,72 @@ func makeInlineOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot,
 		// we only need to prune out overlapping hosts if we have more than
 		// one certificate
 		for _, cert := range certs {
-			hosts, err := cert.Hosts()
-			if err != nil {
-				return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts)
-			}
-			for _, host := range hosts {
-				if _, ok := allCertHosts[host]; ok {
-					overlappingHosts[host] = struct{}{}
+			switch tce := cert.(type) {
+			case *structs.InlineCertificateConfigEntry:
+				hosts, err := tce.Hosts()
+				if err != nil {
+					return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts)
 				}
-				allCertHosts[host] = struct{}{}
+				for _, host := range hosts {
+					if _, ok := allCertHosts[host]; ok {
+						overlappingHosts[host] = struct{}{}
+					}
+					allCertHosts[host] = struct{}{}
+				}
+			default:
+				// do nothing for FileSystemCertificates because we don't actually have the certificate available
 			}
 		}
 	}
 
+	constructTLSContext := func(certConfig structs.ConfigEntry) (*envoy_tls_v3.CommonTlsContext, error) {
+		switch tce := certConfig.(type) {
+		case *structs.InlineCertificateConfigEntry:
+			return makeInlineTLSContextFromGatewayTLSConfig(tlsCfg, tce), nil
+		case *structs.FileSystemCertificateConfigEntry:
+			return makeFileSystemTLSContextFromGatewayTLSConfig(tlsCfg, tce), nil
+		default:
+			return nil, fmt.Errorf("unsupported config entry kind %s", tce.GetKind())
+		}
+	}
+
 	for _, cert := range certs {
 		var hosts []string
 
 		// if we only have one cert, we just use it for all ingress
 		if multipleCerts {
-			// otherwise, we need an SNI per cert and to fallback to our ingress
-			// gateway certificate signed by our Consul CA
-			certHosts, err := cert.Hosts()
-			if err != nil {
-				return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts)
-			}
-			// filter out any overlapping hosts so we don't have collisions in our filter chains
-			for _, host := range certHosts {
-				if _, ok := overlappingHosts[host]; !ok {
-					hosts = append(hosts, host)
+			switch tce := cert.(type) {
+			case *structs.InlineCertificateConfigEntry:
+				certHosts, err := tce.Hosts()
+				if err != nil {
+					return nil, fmt.Errorf("unable to parse hosts from x509 certificate: %v", hosts)
+				}
+				// filter out any overlapping hosts so we don't have collisions in our filter chains
+				for _, host := range certHosts {
+					if _, ok := overlappingHosts[host]; !ok {
+						hosts = append(hosts, host)
+					}
 				}
-			}
 
-			if len(hosts) == 0 {
-				// all of our hosts are overlapping, so we just skip this filter and it'll be
-				// handled by the default filter chain
-				continue
+				if len(hosts) == 0 {
+					// all of our hosts are overlapping, so we just skip this filter and it'll be
+					// handled by the default filter chain
+					continue
+				}
 			}
 		}
 
-		if err := constructChain(cert.Name, hosts, makeInlineTLSContextFromGatewayTLSConfig(tlsCfg, cert)); err != nil {
+		tlsContext, err := constructTLSContext(cert)
+		if err != nil {
+			continue
+		}
+
+		if err := constructChain(cert.GetName(), hosts, tlsContext); err != nil {
 			return nil, err
 		}
 	}
 
-	if multipleCerts {
+	if len(certs) > 1 {
 		// if we have more than one cert, add a default handler that uses the leaf cert from connect
 		if err := constructChain("default", nil, makeCommonTLSContext(cfgSnap.Leaf(), cfgSnap.RootPEMs(), makeTLSParametersFromGatewayTLSConfig(tlsCfg))); err != nil {
 			return nil, err

agent/xds/listeners_ingress.go

@@ -9,7 +9,6 @@ import (
 	envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
-	"github.com/hashicorp/consul/agent/xds/naming"
 
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/durationpb"
@@ -17,6 +16,7 @@ import (
 
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/agent/xds/naming"
 	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/types"
 )
@@ -391,7 +391,26 @@ func makeTLSParametersFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig) *env
 	return makeTLSParametersFromTLSConfig(tlsCfg.TLSMinVersion, tlsCfg.TLSMaxVersion, tlsCfg.CipherSuites)
 }
 
-func makeInlineTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, cert structs.InlineCertificateConfigEntry) *envoy_tls_v3.CommonTlsContext {
+func makeFileSystemTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, cert *structs.FileSystemCertificateConfigEntry) *envoy_tls_v3.CommonTlsContext {
+	return &envoy_tls_v3.CommonTlsContext{
+		TlsParams: makeTLSParametersFromGatewayTLSConfig(tlsCfg),
+		TlsCertificateSdsSecretConfigs: []*envoy_tls_v3.SdsSecretConfig{
+			{
+				// Delivering via SDS is required in order to get file system watches today.
+				//   https://github.com/envoyproxy/envoy/issues/10387
+				Name: cert.Name, // Reference the secret returned in xds/secrets.go by name here
+				SdsConfig: &envoy_core_v3.ConfigSource{
+					ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{
+						Ads: &envoy_core_v3.AggregatedConfigSource{},
+					},
+					ResourceApiVersion: envoy_core_v3.ApiVersion_V3,
+				},
+			},
+		},
+	}
+}
+
+func makeInlineTLSContextFromGatewayTLSConfig(tlsCfg structs.GatewayTLSConfig, cert *structs.InlineCertificateConfigEntry) *envoy_tls_v3.CommonTlsContext {
 	return &envoy_tls_v3.CommonTlsContext{
 		TlsParams: makeTLSParametersFromGatewayTLSConfig(tlsCfg),
 		TlsCertificates: []*envoy_tls_v3.TlsCertificate{{

agent/xds/resources_test.go

@@ -743,6 +743,14 @@ func getMeshGatewayGoldenTestCases() []goldenTestCase {
 			// TODO(proxystate): mesh gateway will come at a later time
 			alsoRunTestForV2: false,
 		},
+		{
+			name: "mesh-gateway-with-limits",
+			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
+				return proxycfg.TestConfigSnapshotMeshGateway(t, "limits-added", nil, nil)
+			},
+			// TODO(proxystate): mesh gateway will come at a later time
+			alsoRunTestForV2: false,
+		},
 		{
 			name: "mesh-gateway-using-federation-states",
 			create: func(t testinf.T) *proxycfg.ConfigSnapshot {

agent/xds/secrets.go

@@ -7,6 +7,8 @@ import (
 	"errors"
 	"fmt"
 
+	envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/hashicorp/consul/agent/proxycfg"
@@ -21,13 +23,45 @@ func (s *ResourceGenerator) secretsFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot
 	}
 
 	switch cfgSnap.Kind {
+	case structs.ServiceKindAPIGateway:
+		return s.secretsFromSnapshotAPIGateway(cfgSnap), nil // return any attached certs
 	case structs.ServiceKindConnectProxy,
 		structs.ServiceKindTerminatingGateway,
 		structs.ServiceKindMeshGateway,
-		structs.ServiceKindIngressGateway,
-		structs.ServiceKindAPIGateway:
+		structs.ServiceKindIngressGateway:
 		return nil, nil
 	default:
 		return nil, fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind)
 	}
 }
+
+// secretsFromSnapshotAPIGateway returns the "secrets" for an api-gateway service
+func (s *ResourceGenerator) secretsFromSnapshotAPIGateway(cfgSnap *proxycfg.ConfigSnapshot) []proto.Message {
+	var resources []proto.Message
+
+	cfgSnap.APIGateway.FileSystemCertificates.ForEachKey(func(ref structs.ResourceReference) bool {
+		cert, ok := cfgSnap.APIGateway.FileSystemCertificates.Get(ref)
+		if !ok || cert == nil {
+			return true
+		}
+		resources = append(resources, &envoy_tls_v3.Secret{
+			Name: ref.Name,
+			Type: &envoy_tls_v3.Secret_TlsCertificate{
+				TlsCertificate: &envoy_tls_v3.TlsCertificate{
+					CertificateChain: &envoy_core_v3.DataSource{
+						Specifier: &envoy_core_v3.DataSource_Filename{
+							Filename: cert.Certificate,
+						}},
+					PrivateKey: &envoy_core_v3.DataSource{
+						Specifier: &envoy_core_v3.DataSource_Filename{
+							Filename: cert.PrivateKey,
+						},
+					},
+				},
+			},
+		})
+		return true
+	})
+
+	return resources
+}

agent/xds/testdata/clusters/mesh-gateway-with-limits.latest.golden

@@ -0,0 +1,151 @@
+{
+  "nonce": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "circuitBreakers": {
+        "thresholds": [
+          {
+            "maxConnections": 1,
+            "maxPendingRequests": 10,
+            "maxRequests": 100
+          }
+        ]
+      },
+      "connectTimeout": "5s",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {},
+          "resourceApiVersion": "V3"
+        }
+      },
+      "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "outlierDetection": {},
+      "type": "EDS"
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "circuitBreakers": {
+        "thresholds": [
+          {
+            "maxConnections": 1,
+            "maxPendingRequests": 10,
+            "maxRequests": 100
+          }
+        ]
+      },
+      "connectTimeout": "5s",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {},
+          "resourceApiVersion": "V3"
+        }
+      },
+      "name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
+      "outlierDetection": {},
+      "type": "EDS"
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "circuitBreakers": {
+        "thresholds": [
+          {
+            "maxConnections": 1,
+            "maxPendingRequests": 10,
+            "maxRequests": 100
+          }
+        ]
+      },
+      "connectTimeout": "5s",
+      "dnsLookupFamily": "V4_ONLY",
+      "dnsRefreshRate": "10s",
+      "loadAssignment": {
+        "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
+        "endpoints": [
+          {
+            "lbEndpoints": [
+              {
+                "endpoint": {
+                  "address": {
+                    "socketAddress": {
+                      "address": "123.us-west-2.elb.notaws.com",
+                      "portValue": 443
+                    }
+                  }
+                },
+                "healthStatus": "HEALTHY",
+                "loadBalancingWeight": 1
+              }
+            ]
+          }
+        ]
+      },
+      "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
+      "outlierDetection": {},
+      "type": "LOGICAL_DNS"
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "circuitBreakers": {
+        "thresholds": [
+          {
+            "maxConnections": 1,
+            "maxPendingRequests": 10,
+            "maxRequests": 100
+          }
+        ]
+      },
+      "connectTimeout": "5s",
+      "dnsLookupFamily": "V4_ONLY",
+      "dnsRefreshRate": "10s",
+      "loadAssignment": {
+        "clusterName": "dc6.internal.11111111-2222-3333-4444-555555555555.consul",
+        "endpoints": [
+          {
+            "lbEndpoints": [
+              {
+                "endpoint": {
+                  "address": {
+                    "socketAddress": {
+                      "address": "123.us-east-1.elb.notaws.com",
+                      "portValue": 443
+                    }
+                  }
+                },
+                "healthStatus": "UNHEALTHY",
+                "loadBalancingWeight": 1
+              }
+            ]
+          }
+        ]
+      },
+      "name": "dc6.internal.11111111-2222-3333-4444-555555555555.consul",
+      "outlierDetection": {},
+      "type": "LOGICAL_DNS"
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "circuitBreakers": {
+        "thresholds": [
+          {
+            "maxConnections": 1,
+            "maxPendingRequests": 10,
+            "maxRequests": 100
+          }
+        ]
+      },
+      "connectTimeout": "5s",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {},
+          "resourceApiVersion": "V3"
+        }
+      },
+      "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "outlierDetection": {},
+      "type": "EDS"
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+  "versionInfo": "00000001"
+}

agent/xds/testdata/endpoints/mesh-gateway-with-limits.latest.golden

@@ -0,0 +1,145 @@
+{
+  "nonce": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+      "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.6",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.7",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.8",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+      "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.1",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.2",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+      "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.3",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.4",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.5",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.9",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+  "versionInfo": "00000001"
+}

agent/xds/testdata/listeners/mesh-gateway-with-limits.latest.golden

@@ -0,0 +1,96 @@
+{
+  "nonce": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
+      "address": {
+        "socketAddress": {
+          "address": "1.2.3.4",
+          "portValue": 8443
+        }
+      },
+      "filterChains": [
+        {
+          "filterChainMatch": {
+            "serverNames": [
+              "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul"
+            ]
+          },
+          "filters": [
+            {
+              "name": "envoy.filters.network.tcp_proxy",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+                "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
+                "statPrefix": "mesh_gateway_remote.default.dc2"
+              }
+            }
+          ]
+        },
+        {
+          "filterChainMatch": {
+            "serverNames": [
+              "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul"
+            ]
+          },
+          "filters": [
+            {
+              "name": "envoy.filters.network.tcp_proxy",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+                "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
+                "statPrefix": "mesh_gateway_remote.default.dc4"
+              }
+            }
+          ]
+        },
+        {
+          "filterChainMatch": {
+            "serverNames": [
+              "*.dc6.internal.11111111-2222-3333-4444-555555555555.consul"
+            ]
+          },
+          "filters": [
+            {
+              "name": "envoy.filters.network.tcp_proxy",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+                "cluster": "dc6.internal.11111111-2222-3333-4444-555555555555.consul",
+                "statPrefix": "mesh_gateway_remote.default.dc6"
+              }
+            }
+          ]
+        },
+        {
+          "filters": [
+            {
+              "name": "envoy.filters.network.sni_cluster",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.sni_cluster.v3.SniCluster"
+              }
+            },
+            {
+              "name": "envoy.filters.network.tcp_proxy",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+                "cluster": "",
+                "statPrefix": "mesh_gateway_local.default"
+              }
+            }
+          ]
+        }
+      ],
+      "listenerFilters": [
+        {
+          "name": "envoy.filters.listener.tls_inspector",
+          "typedConfig": {
+            "@type": "type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector"
+          }
+        }
+      ],
+      "name": "default:1.2.3.4:8443"
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
+  "versionInfo": "00000001"
+}

agent/xds/testdata/routes/mesh-gateway-with-limits.latest.golden

@@ -0,0 +1,5 @@
+{
+  "nonce": "00000001",
+  "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+  "versionInfo": "00000001"
+}

agent/xds/testdata/secrets/mesh-gateway-with-limits.latest.golden

@@ -0,0 +1,5 @@
+{
+  "nonce": "00000001",
+  "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
+  "versionInfo": "00000001"
+}

api/config_entry.go

@@ -29,13 +29,14 @@ const (
 	SamenessGroup      string = "sameness-group"
 	RateLimitIPConfig  string = "control-plane-request-limit"
 
-	ProxyConfigGlobal string = "global"
-	MeshConfigMesh    string = "mesh"
-	APIGateway        string = "api-gateway"
-	TCPRoute          string = "tcp-route"
-	InlineCertificate string = "inline-certificate"
-	HTTPRoute         string = "http-route"
-	JWTProvider       string = "jwt-provider"
+	ProxyConfigGlobal     string = "global"
+	MeshConfigMesh        string = "mesh"
+	APIGateway            string = "api-gateway"
+	TCPRoute              string = "tcp-route"
+	FileSystemCertificate string = "file-system-certificate"
+	InlineCertificate     string = "inline-certificate"
+	HTTPRoute             string = "http-route"
+	JWTProvider           string = "jwt-provider"
 )
 
 const (
@@ -447,6 +448,8 @@ func makeConfigEntry(kind, name string) (ConfigEntry, error) {
 		return &APIGatewayConfigEntry{Kind: kind, Name: name}, nil
 	case TCPRoute:
 		return &TCPRouteConfigEntry{Kind: kind, Name: name}, nil
+	case FileSystemCertificate:
+		return &FileSystemCertificateConfigEntry{Kind: kind, Name: name}, nil
 	case InlineCertificate:
 		return &InlineCertificateConfigEntry{Kind: kind, Name: name}, nil
 	case HTTPRoute:

api/config_entry_file_system_certificate.go

@@ -0,0 +1,44 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package api
+
+type FileSystemCertificateConfigEntry struct {
+	// Kind of the config entry. This should be set to api.FileSystemCertificate.
+	Kind string
+
+	Name string
+
+	// Certificate is the path to a client certificate to use for TLS connections.
+	Certificate string `json:",omitempty" alias:"certificate"`
+
+	// PrivateKey is the path to a private key to use for TLS connections.
+	PrivateKey string `json:",omitempty" alias:"private_key"`
+
+	Meta map[string]string `json:",omitempty"`
+
+	// CreateIndex is the Raft index this entry was created at. This is a
+	// read-only field.
+	CreateIndex uint64
+
+	// ModifyIndex is used for the Check-And-Set operations and can also be fed
+	// back into the WaitIndex of the QueryOptions in order to perform blocking
+	// queries.
+	ModifyIndex uint64
+
+	// Partition is the partition the config entry is associated with.
+	// Partitioning is a Consul Enterprise feature.
+	Partition string `json:",omitempty"`
+
+	// Namespace is the namespace the config entry is associated with.
+	// Namespacing is a Consul Enterprise feature.
+	Namespace string `json:",omitempty"`
+}
+
+func (a *FileSystemCertificateConfigEntry) GetKind() string            { return FileSystemCertificate }
+func (a *FileSystemCertificateConfigEntry) GetName() string            { return a.Name }
+func (a *FileSystemCertificateConfigEntry) GetPartition() string       { return a.Partition }
+func (a *FileSystemCertificateConfigEntry) GetNamespace() string       { return a.Namespace }
+func (a *FileSystemCertificateConfigEntry) GetMeta() map[string]string { return a.Meta }
+func (a *FileSystemCertificateConfigEntry) GetCreateIndex() uint64     { return a.CreateIndex }
+func (a *FileSystemCertificateConfigEntry) GetModifyIndex() uint64     { return a.ModifyIndex }

envoyextensions/xdscommon/xdscommon.go

@@ -8,6 +8,7 @@ import (
 	envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"github.com/hashicorp/go-hclog"
 	"google.golang.org/protobuf/proto"
 )
@@ -111,7 +112,7 @@ func IndexResources(logger hclog.Logger, resources map[string][]proto.Message) *
 }
 
 func GetResourceName(res proto.Message) string {
-	// NOTE: this only covers types that we currently care about for LDS/RDS/CDS/EDS
+	// NOTE: this only covers types that we currently care about for LDS/RDS/CDS/EDS/SDS
 	switch x := res.(type) {
 	case *envoy_listener_v3.Listener: // LDS
 		return x.Name
@@ -121,6 +122,8 @@ func GetResourceName(res proto.Message) string {
 		return x.Name
 	case *envoy_endpoint_v3.ClusterLoadAssignment: // EDS
 		return x.ClusterName
+	case *envoy_tls_v3.Secret: // SDS
+		return x.Name
 	default:
 		return ""
 	}
@@ -133,6 +136,7 @@ func EmptyIndexedResources() *IndexedResources {
 			RouteType:    make(map[string]proto.Message),
 			ClusterType:  make(map[string]proto.Message),
 			EndpointType: make(map[string]proto.Message),
+			SecretType:   make(map[string]proto.Message),
 		},
 		ChildIndex: map[string]map[string][]string{
 			ListenerType: make(map[string][]string),

lib/telemetry.go

@@ -146,9 +146,14 @@ type TelemetryConfig struct {
 
 	// DisableHostname will disable hostname prefixing for all metrics.
 	//
-	// hcl: telemetry { disable_hostname = (true|false)
+	// hcl: telemetry { disable_hostname = (true|false) }
 	DisableHostname bool `json:"disable_hostname,omitempty" mapstructure:"disable_hostname"`
 
+	// DisablePerTenancyUsageMetrics will disable setting tenancy labels on usage metrics.
+	//
+	// hcl: telemetry { disable_per_tenancy_usage_metrics = (true|false) }
+	DisablePerTenancyUsageMetrics bool `json:"disable_per_tenancy_usage_metrics,omitempty" mapstructure:"disable_per_tenancy_usage_metrics"`
+
 	// DogStatsdAddr is the address of a dogstatsd instance. If provided,
 	// metrics will be sent to that instance
 	//

proto/private/pbconfigentry/config_entry.gen.go

@@ -408,6 +408,24 @@ func ExposePathFromStructs(t *structs.ExposePath, s *ExposePath) {
 	s.Protocol = t.Protocol
 	s.ParsedFromCheck = t.ParsedFromCheck
 }
+func FileSystemCertificateToStructs(s *FileSystemCertificate, t *structs.FileSystemCertificateConfigEntry) {
+	if s == nil {
+		return
+	}
+	t.Certificate = s.Certificate
+	t.PrivateKey = s.PrivateKey
+	t.Meta = s.Meta
+	t.Hash = s.Hash
+}
+func FileSystemCertificateFromStructs(t *structs.FileSystemCertificateConfigEntry, s *FileSystemCertificate) {
+	if s == nil {
+		return
+	}
+	s.Certificate = t.Certificate
+	s.PrivateKey = t.PrivateKey
+	s.Meta = t.Meta
+	s.Hash = t.Hash
+}
 func GatewayServiceTLSConfigToStructs(s *GatewayServiceTLSConfig, t *structs.GatewayServiceTLSConfig) {
 	if s == nil {
 		return

proto/private/pbconfigentry/config_entry.go

@@ -85,6 +85,14 @@ func ConfigEntryToStructs(s *ConfigEntry) structs.ConfigEntry {
 		pbcommon.RaftIndexToStructs(s.RaftIndex, &target.RaftIndex)
 		pbcommon.EnterpriseMetaToStructs(s.EnterpriseMeta, &target.EnterpriseMeta)
 		return &target
+	case Kind_KindFileSystemCertificate:
+		var target structs.FileSystemCertificateConfigEntry
+		target.Name = s.Name
+
+		FileSystemCertificateToStructs(s.GetFileSystemCertificate(), &target)
+		pbcommon.RaftIndexToStructs(s.RaftIndex, &target.RaftIndex)
+		pbcommon.EnterpriseMetaToStructs(s.EnterpriseMeta, &target.EnterpriseMeta)
+		return &target
 	case Kind_KindInlineCertificate:
 		var target structs.InlineCertificateConfigEntry
 		target.Name = s.Name
@@ -213,6 +221,14 @@ func ConfigEntryFromStructs(s structs.ConfigEntry) *ConfigEntry {
 		configEntry.Entry = &ConfigEntry_HTTPRoute{
 			HTTPRoute: &route,
 		}
+	case *structs.FileSystemCertificateConfigEntry:
+		var cert FileSystemCertificate
+		FileSystemCertificateFromStructs(v, &cert)
+
+		configEntry.Kind = Kind_KindFileSystemCertificate
+		configEntry.Entry = &ConfigEntry_FileSystemCertificate{
+			FileSystemCertificate: &cert,
+		}
 	case *structs.InlineCertificateConfigEntry:
 		var cert InlineCertificate
 		InlineCertificateFromStructs(v, &cert)

proto/private/pbconfigentry/config_entry.pb.binary.go

@@ -657,6 +657,16 @@ func (msg *BoundAPIGatewayListener) UnmarshalBinary(b []byte) error {
 	return proto.Unmarshal(b, msg)
 }
 
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *FileSystemCertificate) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *FileSystemCertificate) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
 // MarshalBinary implements encoding.BinaryMarshaler
 func (msg *InlineCertificate) MarshalBinary() ([]byte, error) {
 	return proto.Marshal(msg)

proto/private/pbconfigentry/config_entry.proto

@@ -54,6 +54,7 @@ enum Kind {
   KindSamenessGroup = 11;
   KindJWTProvider = 12;
   KindExportedServices = 13;
+  KindFileSystemCertificate = 14;
 }
 
 message ConfigEntry {
@@ -77,6 +78,7 @@ message ConfigEntry {
     SamenessGroup SamenessGroup = 15;
     JWTProvider JWTProvider = 16;
     ExportedServices ExportedServices = 17;
+    FileSystemCertificate FileSystemCertificate = 18;
   }
 }
 
@@ -867,6 +869,19 @@ message BoundAPIGatewayListener {
   repeated ResourceReference Routes = 3;
 }
 
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.FileSystemCertificateConfigEntry
+// output=config_entry.gen.go
+// name=Structs
+// ignore-fields=Kind,Name,RaftIndex,EnterpriseMeta
+message FileSystemCertificate {
+  map<string, string> Meta = 1;
+  string Certificate = 2;
+  string PrivateKey = 3;
+  uint64 Hash = 4;
+}
+
 // mog annotation:
 //
 // target=github.com/hashicorp/consul/agent/structs.InlineCertificateConfigEntry

proto/private/pbsubscribe/subscribe.pb.go

@@ -80,6 +80,8 @@ const (
 	Topic_JWTProvider Topic = 16
 	// ExportedServices topic contains events for changes to exported-services.
 	Topic_ExportedServices Topic = 17
+	// FileSystemCertificate topic contains events for changes to file-system-certificates.
+	Topic_FileSystemCertificate Topic = 18
 )
 
 // Enum value maps for Topic.
@@ -103,26 +105,28 @@ var (
 		15: "SamenessGroup",
 		16: "JWTProvider",
 		17: "ExportedServices",
+		18: "FileSystemCertificate",
 	}
 	Topic_value = map[string]int32{
-		"Unknown":              0,
-		"ServiceHealth":        1,
-		"ServiceHealthConnect": 2,
-		"MeshConfig":           3,
-		"ServiceResolver":      4,
-		"IngressGateway":       5,
-		"ServiceIntentions":    6,
-		"ServiceList":          7,
-		"ServiceDefaults":      8,
-		"APIGateway":           9,
-		"TCPRoute":             10,
-		"HTTPRoute":            11,
-		"InlineCertificate":    12,
-		"BoundAPIGateway":      13,
-		"IPRateLimit":          14,
-		"SamenessGroup":        15,
-		"JWTProvider":          16,
-		"ExportedServices":     17,
+		"Unknown":               0,
+		"ServiceHealth":         1,
+		"ServiceHealthConnect":  2,
+		"MeshConfig":            3,
+		"ServiceResolver":       4,
+		"IngressGateway":        5,
+		"ServiceIntentions":     6,
+		"ServiceList":           7,
+		"ServiceDefaults":       8,
+		"APIGateway":            9,
+		"TCPRoute":              10,
+		"HTTPRoute":             11,
+		"InlineCertificate":     12,
+		"BoundAPIGateway":       13,
+		"IPRateLimit":           14,
+		"SamenessGroup":         15,
+		"JWTProvider":           16,
+		"ExportedServices":      17,
+		"FileSystemCertificate": 18,
 	}
 )
 
@@ -1004,7 +1008,7 @@ var file_private_pbsubscribe_subscribe_proto_rawDesc = []byte{
 	0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x0e, 0x45,
 	0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x1a, 0x0a,
 	0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x2a, 0xdb, 0x02, 0x0a, 0x05, 0x54, 0x6f,
+	0x08, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x2a, 0xf6, 0x02, 0x0a, 0x05, 0x54, 0x6f,
 	0x70, 0x69, 0x63, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00,
 	0x12, 0x11, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74,
 	0x68, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x48, 0x65,
@@ -1026,26 +1030,28 @@ var file_private_pbsubscribe_subscribe_proto_rawDesc = []byte{
 	0x0d, 0x53, 0x61, 0x6d, 0x65, 0x6e, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x10, 0x0f,
 	0x12, 0x0f, 0x0a, 0x0b, 0x4a, 0x57, 0x54, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x10,
 	0x10, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x53, 0x65, 0x72,
-	0x76, 0x69, 0x63, 0x65, 0x73, 0x10, 0x11, 0x2a, 0x29, 0x0a, 0x09, 0x43, 0x61, 0x74, 0x61, 0x6c,
-	0x6f, 0x67, 0x4f, 0x70, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72,
-	0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x65, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72,
-	0x10, 0x01, 0x32, 0x61, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x6e, 0x67,
-	0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a,
-	0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x1b, 0x2e, 0x73, 0x75, 0x62,
-	0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72,
-	0x69, 0x62, 0x65, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08,
-	0x02, 0x10, 0x09, 0x30, 0x01, 0x42, 0x9a, 0x01, 0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x2e, 0x73, 0x75,
-	0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x42, 0x0e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
-	0x62, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f,
-	0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x72, 0x69,
-	0x76, 0x61, 0x74, 0x65, 0x2f, 0x70, 0x62, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65,
-	0xa2, 0x02, 0x03, 0x53, 0x58, 0x58, 0xaa, 0x02, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
-	0x62, 0x65, 0xca, 0x02, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0xe2, 0x02,
-	0x15, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65,
-	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
-	0x62, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x76, 0x69, 0x63, 0x65, 0x73, 0x10, 0x11, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x69, 0x6c, 0x65, 0x53,
+	0x79, 0x73, 0x74, 0x65, 0x6d, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65,
+	0x10, 0x12, 0x2a, 0x29, 0x0a, 0x09, 0x43, 0x61, 0x74, 0x61, 0x6c, 0x6f, 0x67, 0x4f, 0x70, 0x12,
+	0x0c, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x10, 0x00, 0x12, 0x0e, 0x0a,
+	0x0a, 0x44, 0x65, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x10, 0x01, 0x32, 0x61, 0x0a,
+	0x17, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x75, 0x62, 0x73,
+	0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x73,
+	0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x1b, 0x2e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62,
+	0x65, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x2e, 0x45,
+	0x76, 0x65, 0x6e, 0x74, 0x22, 0x08, 0xe2, 0x86, 0x04, 0x04, 0x08, 0x02, 0x10, 0x09, 0x30, 0x01,
+	0x42, 0x9a, 0x01, 0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x2e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
+	0x62, 0x65, 0x42, 0x0e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50, 0x72, 0x6f,
+	0x74, 0x6f, 0x50, 0x01, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+	0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75,
+	0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x2f,
+	0x70, 0x62, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0xa2, 0x02, 0x03, 0x53, 0x58,
+	0x58, 0xaa, 0x02, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0xca, 0x02, 0x09,
+	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0xe2, 0x02, 0x15, 0x53, 0x75, 0x62, 0x73,
+	0x63, 0x72, 0x69, 0x62, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
+	0x61, 0xea, 0x02, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x62, 0x06, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

proto/private/pbsubscribe/subscribe.proto

@@ -109,6 +109,9 @@ enum Topic {
 
   // ExportedServices topic contains events for changes to exported-services.
   ExportedServices = 17;
+
+  // FileSystemCertificate topic contains events for changes to file-system-certificates.
+  FileSystemCertificate = 18;
 }
 
 message NamedSubject {

test/integration/connect/envoy/run-tests.sh

@@ -227,7 +227,7 @@ function start_consul {
     docker_kill_rm consul-${DC}-server
     docker_kill_rm consul-${DC}
 
-    docker run -d --name envoy_consul-${DC}-server_1 \
+    docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name envoy_consul-${DC}-server_1 \
       --net=envoy-tests \
       $WORKDIR_SNIPPET \
       --hostname "consul-${DC}-server" \
@@ -240,7 +240,7 @@ function start_consul {
       -client "0.0.0.0" \
       -bind "0.0.0.0" >/dev/null
 
-    docker run -d --name envoy_consul-${DC}_1 \
+    docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name envoy_consul-${DC}_1 \
       --net=envoy-tests \
       $WORKDIR_SNIPPET \
       --hostname "consul-${DC}-client" \
@@ -258,7 +258,7 @@ function start_consul {
   else
     docker_kill_rm consul-${DC}
 
-    docker run -d --name envoy_consul-${DC}_1 \
+    docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name envoy_consul-${DC}_1 \
       --net=envoy-tests \
       $WORKDIR_SNIPPET \
       --hostname "consul-${DC}" \
@@ -295,7 +295,7 @@ function start_partitioned_client {
   # Run consul and expose some ports to the host to make debugging locally a
   # bit easier.
   #
-  docker run -d --name envoy_consul-${PARTITION}_1 \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name envoy_consul-${PARTITION}_1 \
     --net=envoy-tests \
     $WORKDIR_SNIPPET \
     --hostname "consul-${PARTITION}-client" \
@@ -351,7 +351,7 @@ function verify {
 
   # need to tell the PID 1 inside of the container that it won't be actual PID
   # 1 because we're using --pid=host so we use TINI_SUBREAPER
-  if docker run --name envoy_verify-${CLUSTER}_1 -t \
+  if docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 --name envoy_verify-${CLUSTER}_1 -t \
     -e TINI_SUBREAPER=1 \
     -e ENVOY_VERSION \
     $WORKDIR_SNIPPET \
@@ -442,7 +442,7 @@ function global_setup {
 }
 
 function wipe_volumes {
-  docker run --rm -i \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 --rm -i \
     $WORKDIR_SNIPPET \
     --net=none \
     "${HASHICORP_DOCKER_PROXY}/alpine" \
@@ -556,7 +556,7 @@ function suite_setup {
     # This is a dummy container that we use to create volume and keep it
     # accessible while other containers are down.
     docker volume create envoy_workdir &>/dev/null
-    docker run -d --name envoy_workdir_1 \
+    docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name envoy_workdir_1 \
         $WORKDIR_SNIPPET \
         --net=none \
         k8s.gcr.io/pause &>/dev/null
@@ -567,7 +567,7 @@ function suite_setup {
     retry_default docker build -t bats-verify -f Dockerfile-bats .
 
     echo "Checking bats image..."
-    docker run --rm -t bats-verify -v
+    docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 --rm -t bats-verify -v
 
     # pre-build the consul+envoy container
     echo "Rebuilding 'consul-dev-envoy:${ENVOY_VERSION}' image..."
@@ -615,7 +615,7 @@ function common_run_container_service {
   local httpPort="$3"
   local grpcPort="$4"
 
-  docker run -d --name $(container_name_prev) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name_prev) \
     -e "FORTIO_NAME=${service}" \
     $(network_snippet $CLUSTER) \
     "${HASHICORP_DOCKER_PROXY}/fortio/fortio" \
@@ -696,7 +696,7 @@ function common_run_container_sidecar_proxy {
   # despite separate containers that don't share IPC namespace. Not quite
   # sure how this happens but may be due to unix socket being in some shared
   # location?
-  docker run -d --name $(container_name_prev) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name_prev) \
     $WORKDIR_SNIPPET \
     $(network_snippet $CLUSTER) \
     $(aws_snippet) \
@@ -717,7 +717,7 @@ function run_container_s1-ap1-sidecar-proxy {
 }
 
 function run_container_s1-sidecar-proxy-consul-exec {
-  docker run -d --name $(container_name) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name) \
     $(network_snippet primary) \
     consul-dev-envoy:${ENVOY_VERSION} \
     consul connect envoy -sidecar-for s1 \
@@ -783,7 +783,7 @@ function common_run_container_gateway {
   # despite separate containers that don't share IPC namespace. Not quite
   # sure how this happens but may be due to unix socket being in some shared
   # location?
-  docker run -d --name $(container_name_prev) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name_prev) \
     $WORKDIR_SNIPPET \
     $(network_snippet $DC) \
     $(aws_snippet) \
@@ -824,7 +824,7 @@ function run_container_fake-statsd {
   # This magic SYSTEM incantation is needed since Envoy doesn't add newlines and so
   # we need each packet to be passed to echo to add a new line before
   # appending.
-  docker run -d --name $(container_name) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name) \
     $WORKDIR_SNIPPET \
     $(network_snippet primary) \
     "${HASHICORP_DOCKER_PROXY}/alpine/socat:1.7.3.4-r1" \
@@ -833,14 +833,14 @@ function run_container_fake-statsd {
 }
 
 function run_container_zipkin {
-  docker run -d --name $(container_name) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name) \
     $WORKDIR_SNIPPET \
     $(network_snippet primary) \
     "${HASHICORP_DOCKER_PROXY}/openzipkin/zipkin"
 }
 
 function run_container_jaeger {
-  docker run -d --name $(container_name) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name) \
     $WORKDIR_SNIPPET \
     $(network_snippet primary) \
     "${HASHICORP_DOCKER_PROXY}/jaegertracing/all-in-one:1.11" \
@@ -848,7 +848,7 @@ function run_container_jaeger {
 }
 
 function run_container_test-sds-server {
-  docker run -d --name $(container_name) \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name) \
     $WORKDIR_SNIPPET \
     $(network_snippet primary) \
     "test-sds-server"
@@ -863,7 +863,7 @@ function container_name_prev {
 
 # This is a debugging tool. Run via './run-tests.sh debug_dump_volumes'
 function debug_dump_volumes {
-  docker run --rm -it \
+  docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 --rm -it \
     $WORKDIR_SNIPPET \
     -v ./:/cwd \
     --net=none \
@@ -891,7 +891,7 @@ function common_run_container_tcpdump {
 
     retry_default docker build -t envoy-tcpdump -f Dockerfile-tcpdump .
 
-    docker run -d --name $(container_name_prev) \
+    docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name $(container_name_prev) \
         $(network_snippet $DC) \
         -v $(pwd)/workdir/${DC}/envoy/:/data \
         --privileged \

website/content/api-docs/config.mdx

@@ -39,23 +39,24 @@ The corresponding CLI command is [`consul config write`](/consul/commands/config
 
 The ACL required depends on the config entry being written:
 
-| Config Entry Kind   | Required ACLs                    |
-| ------------------- | -------------------------------- |
-| api-gateway         | `mesh:write` or `operator:write` |
-| bound-api-gateway   | Not writable.                    |
-| exported-services   | `mesh:write` or `operator:write` |
-| http-route          | `mesh:write` or `operator:write` |
-| ingress-gateway     | `mesh:write` or `operator:write` |
-| inline-certificate  | `mesh:write` or `operator:write` |
-| mesh                | `mesh:write` or `operator:write` |
-| proxy-defaults      | `mesh:write` or `operator:write` |
-| service-defaults    | `service:write`                  |
-| service-intentions  | `intentions:write`               |
-| service-resolver    | `service:write`                  |
-| service-router      | `service:write`                  |
-| service-splitter    | `service:write`                  |
-| tcp-route           | `mesh:write` or `operator:write` |
-| terminating-gateway | `mesh:write` or `operator:write` |
+| Config Entry Kind       | Required ACLs                    |
+| ----------------------- | -------------------------------- |
+| api-gateway             | `mesh:write` or `operator:write` |
+| bound-api-gateway       | Not writable.                    |
+| exported-services       | `mesh:write` or `operator:write` |
+| file-system-certificate | `mesh:write` or `operator:write` |
+| http-route              | `mesh:write` or `operator:write` |
+| ingress-gateway         | `mesh:write` or `operator:write` |
+| inline-certificate      | `mesh:write` or `operator:write` |
+| mesh                    | `mesh:write` or `operator:write` |
+| proxy-defaults          | `mesh:write` or `operator:write` |
+| service-defaults        | `service:write`                  |
+| service-intentions      | `intentions:write`               |
+| service-resolver        | `service:write`                  |
+| service-router          | `service:write`                  |
+| service-splitter        | `service:write`                  |
+| tcp-route               | `mesh:write` or `operator:write` |
+| terminating-gateway     | `mesh:write` or `operator:write` |
 
 ### Query Parameters
 
@@ -114,23 +115,24 @@ The corresponding CLI command is [`consul config read`](/consul/commands/config/
 
 The ACL required depends on the config entry kind being read:
 
-| Config Entry Kind   | Required ACLs                    |
-| ------------------- | -------------------------------- |
-| api-gateway         | `service:read`                   |
-| bound-api-gateway   | `service:read`                   |
-| exported-services   | `mesh:read` or `operator:read`   |
-| http-route          | `mesh:read` or `operator:read`   |
-| ingress-gateway     | `service:read`                   |
-| inline-certificate  | `mesh:read` or `operator:read`   |
-| mesh                | No ACL required                  |
-| proxy-defaults      | No ACL required                  |
-| service-defaults    | `service:read`                   |
-| service-intentions  | `intentions:read`                |
-| service-resolver    | `service:read`                   |
-| service-router      | `service:read`                   |
-| service-splitter    | `service:read`                   |
-| tcp-route           | `mesh:read` or `operator:read`   |
-| terminating-gateway | `service:read`                   |
+| Config Entry Kind       | Required ACLs                    |
+| ----------------------- | -------------------------------- |
+| api-gateway             | `service:read`                   |
+| bound-api-gateway       | `service:read`                   |
+| exported-services       | `mesh:read` or `operator:read`   |
+| file-system-certificate | `mesh:read` or `operator:read` |
+| http-route              | `mesh:read` or `operator:read`   |
+| ingress-gateway         | `service:read`                   |
+| inline-certificate      | `mesh:read` or `operator:read`   |
+| mesh                    | No ACL required                  |
+| proxy-defaults          | No ACL required                  |
+| service-defaults        | `service:read`                   |
+| service-intentions      | `intentions:read`                |
+| service-resolver        | `service:read`                   |
+| service-router          | `service:read`                   |
+| service-splitter        | `service:read`                   |
+| tcp-route               | `mesh:read` or `operator:read`   |
+| terminating-gateway     | `service:read`                   |
 
 ### Path Parameters
 
@@ -192,23 +194,24 @@ The table below shows this endpoint's support for
 
 The ACL required depends on the config entry kind being read:
 
-| Config Entry Kind   | Required ACLs                    |
-| ------------------- | -------------------------------- |
-| api-gateway         | `service:read`                   |
-| bound-api-gateway   | `service:read`                   |
-| exported-services   | `mesh:read` or `operator:read`   |
-| http-route          | `mesh:read` or `operator:read`   |
-| ingress-gateway     | `service:read`                   |
-| inline-certificate  | `mesh:read` or `operator:read`   |
-| mesh                | No ACL required                  |
-| proxy-defaults      | No ACL required                  |
-| service-defaults    | `service:read`                   |
-| service-intentions  | `intentions:read`                |
-| service-resolver    | `service:read`                   |
-| service-router      | `service:read`                   |
-| service-splitter    | `service:read`                   |
-| tcp-route           | `mesh:read` or `operator:read`   |
-| terminating-gateway | `service:read`                   |
+| Config Entry Kind       | Required ACLs                    |
+| ----------------------- | -------------------------------- |
+| api-gateway             | `service:read`                   |
+| bound-api-gateway       | `service:read`                   |
+| exported-services       | `mesh:read` or `operator:read`   |
+| file-system-certificate | `mesh:read` or `operator:read` |
+| http-route              | `mesh:read` or `operator:read`   |
+| ingress-gateway         | `service:read`                   |
+| inline-certificate      | `mesh:read` or `operator:read`   |
+| mesh                    | No ACL required                  |
+| proxy-defaults          | No ACL required                  |
+| service-defaults        | `service:read`                   |
+| service-intentions      | `intentions:read`                |
+| service-resolver        | `service:read`                   |
+| service-router          | `service:read`                   |
+| service-splitter        | `service:read`                   |
+| tcp-route               | `mesh:read` or `operator:read`   |
+| terminating-gateway     | `service:read`                   |
 
 The corresponding CLI command is [`consul config list`](/consul/commands/config/list).
 
@@ -276,23 +279,24 @@ The table below shows this endpoint's support for
 
 The ACL required depends on the config entry kind being deleted:
 
-| Config Entry Kind   | Required ACLs                    |
-| ------------------- | -------------------------------- |
-| api-gateway         | `mesh:write` or `operator:write` |
-| bound-api-gateway   | Not writable.                    |
-| exported-services   | `mesh:write` or `operator:write` |
-| http-route          | `mesh:write` or `operator:write` |
-| ingress-gateway     | `mesh:write` or `operator:write` |
-| inline-certificate  | `mesh:write` or `operator:write` |
-| mesh                | `mesh:write` or `operator:write` |
-| proxy-defaults      | `mesh:write` or `operator:write` |
-| service-defaults    | `service:write`                  |
-| service-intentions  | `intentions:write`               |
-| service-resolver    | `service:write`                  |
-| service-router      | `service:write`                  |
-| service-splitter    | `service:write`                  |
-| tcp-route           | `mesh:write` or `operator:write` |
-| terminating-gateway | `mesh:write` or `operator:write` |
+| Config Entry Kind       | Required ACLs                    |
+| ----------------------- | -------------------------------- |
+| api-gateway             | `mesh:write` or `operator:write` |
+| bound-api-gateway       | Not writable.                    |
+| exported-services       | `mesh:write` or `operator:write` |
+| file-system-certificate | `mesh:write` or `operator:write` |
+| http-route              | `mesh:write` or `operator:write` |
+| ingress-gateway         | `mesh:write` or `operator:write` |
+| inline-certificate      | `mesh:write` or `operator:write` |
+| mesh                    | `mesh:write` or `operator:write` |
+| proxy-defaults          | `mesh:write` or `operator:write` |
+| service-defaults        | `service:write`                  |
+| service-intentions      | `intentions:write`               |
+| service-resolver        | `service:write`                  |
+| service-router          | `service:write`                  |
+| service-splitter        | `service:write`                  |
+| tcp-route               | `mesh:write` or `operator:write` |
+| terminating-gateway     | `mesh:write` or `operator:write` |
 
 The corresponding CLI command is [`consul config delete`](/consul/commands/config/delete).
 

website/content/docs/agent/config/config-files.mdx

@@ -1301,7 +1301,7 @@ subsystem that provides Consul's service mesh capabilities.
 ## DNS and Domain Parameters
 
 - `dns_config` This object allows a number of sub-keys
-  to be set which can tune how DNS queries are serviced. Check the tutorial on [DNS caching](/consul/tutorials/networking/dns-caching) for more detail.
+  to be set which can tune how DNS queries are serviced. Refer to [DNS caching](/consul/docs/services/discovery/dns-cache) for more information.
 
   The following sub-keys are available:
 
@@ -1842,6 +1842,9 @@ subsystem that provides Consul's service mesh capabilities.
   - `disable_hostname` ((#telemetry-disable_hostname))
     Set to `true` to stop prepending the machine's hostname to gauge-type metrics. Default is `false`.
 
+  - `disable_per_tenancy_usage_metrics` ((#telemetry-disable_per_tenancy_usage_metrics))
+    Set to `true` to exclude tenancy labels from usage metrics. This significantly decreases CPU utilization in clusters with many admin partitions or namespaces.
+
   - `dogstatsd_addr` ((#telemetry-dogstatsd_addr)) This provides the address
     of a DogStatsD instance in the format `host:port`. DogStatsD is a protocol-compatible
     flavor of statsd, with the added ability to decorate metrics with tags and event

website/content/docs/architecture/scale.mdx

@@ -126,7 +126,7 @@ We strongly recommend using [stale consistency mode for DNS lookups](/consul/api
 
 We also recommend that you do not configure [`dns_config.max_stale` to limit the staleness of DNS responses](/consul/api-docs/features/consistency#limiting-staleness-advanced-usage), as it may result in a prolonged outage if your Consul servers become overloaded. If bounded result consistency is required by a service, consider modifying the service to use consistent service discovery HTTP API queries instead of DNS lookups.
 
-Avoid using [`dns_config.use_cache`](/consul/docs/agent/config/config-files#dns_use_cache) when operating Consul at scale. Because the Consul agent cache allocates memory for each requested route and each allocation can live up to 3 days, severe memory issues may occur. To implement DNS caching, we instead recommend that you [configure TTLs for services and nodes](/consul/tutorials/networking/dns-caching#ttl) to enable the DNS client to cache responses from Consul.
+Avoid using [`dns_config.use_cache`](/consul/docs/agent/config/config-files#dns_use_cache) when operating Consul at scale. Because the Consul agent cache allocates memory for each requested route and each allocation can live up to 3 days, severe memory issues may occur. To implement DNS caching, we instead recommend that you [configure TTLs for services and nodes](/consul/docs/services/discovery/dns-cache#ttl) to enable the DNS client to cache responses from Consul.
 
 #### HTTP API
 

website/content/docs/connect/config-entries/api-gateway.mdx

@@ -31,19 +31,19 @@ The following list outlines field hierarchy, language-specific data types, and r
     - [`MaxVersion`](#listeners-tls-maxversion): string | no default
     - [`CipherSuites`](#listeners-tls-ciphersuites): list of strings | Envoy default cipher suites
     - [`Certificates`](#listeners-tls-certificates): list of objects | no default
-      - [`Kind`](#listeners-tls-certificates-kind): string | must be `"inline-certificate"`
+      - [`Kind`](#listeners-tls-certificates-kind): string | no default
       - [`Name`](#listeners-tls-certificates-name): string | no default
       - [`Namespace`](#listeners-tls-certificates-namespace): string | no default <EnterpriseAlert inline />
       - [`Partition`](#listeners-tls-certificates-partition): string | no default <EnterpriseAlert inline />
   - [`default`](#listeners-default): map
-    - [`JWT`](#listeners-default-jwt): map 
+    - [`JWT`](#listeners-default-jwt): map
       - [`Providers`](#listeners-default-jwt-providers): list
         - [`Name`](#listeners-default-jwt-providers): string
           - [`VerifyClaims`](#listeners-default-jwt-providers): map
               - [`Path`](#listeners-default-jwt-providers): list
               - [`Value`](#listeners-default-jwt-providers): string
   - [`override`](#listeners-override): map
-    - [`JWT`](#listeners-override-jwt): map 
+    - [`JWT`](#listeners-override-jwt): map
       - [`Providers`](#listeners-override-jwt-providers): list
         - [`Name`](#listeners-override-jwt-providers): string
           - [`VerifyClaims`](#listeners-override-jwt-providers): map
@@ -81,8 +81,8 @@ Listeners = [
       ]
       Certificates = [
         {
-          Kind = "inline-certificate"
-          Name = "<name of inline-certificate>"
+          Kind = "file-system-certificate"
+          Name = "<name of file-system-certificate>"
           Namespace = "<enterprise: namespace of the certificate>"
           Partition = "<enterprise: partition of the certificate>"
         }
@@ -97,7 +97,7 @@ Listeners = [
           Value = "<value of claim>"
         }
       ]
-     } 
+     }
     }
     override = {
      JWT = {
@@ -108,7 +108,7 @@ Listeners = [
           Value = "<value of claim>"
         }
       ]
-     } 
+     }
     }
   }
 ]
@@ -136,8 +136,8 @@ Listeners = [
         ],
         "Certificates": [
           {
-            "Kind": "inline-certificate",
-            "Name": "<name of inline-certificate>",
+            "Kind": "file-system-certificate",
+            "Name": "<name of file-system-certificate>",
             "Namespace": "<enterprise: namespace of the certificate>",
             "Partition": "<enterprise: partition of the certificate>"
           }
@@ -349,7 +349,7 @@ Specifies a list of cipher suites that the listener supports when negotiating co
 
 ### `Listeners[].TLS.Certificates[]`
 
-The list of references to inline certificates that the listener uses for TLS termination.
+The list of references to file system or inline certificates that the listener uses for TLS termination.
 
 #### Values
 
@@ -362,17 +362,17 @@ The list of references to inline certificates that the listener uses for TLS ter
 
 ### `Listeners[].TLS.Certificates[].Kind`
 
-The list of references to inline-certificates that the listener uses for TLS termination.
+The list of references to certificates that the listener uses for TLS termination.
 
 #### Values
 
 - Default: None
-- This field is required and must be set to `"inline-certificate"`.
-- Data type: string
+- This field is required.
+- The data type is one of the following string values: `"file-system-certificate"` or `"inline-certificate"`.
 
 ### `Listeners[].TLS.Certificates[].Name`
 
-The list of references to inline certificates that the listener uses for TLS termination.
+Specifies the name of the file system or inline certificate that the listener uses for TLS termination.
 
 #### Values
 
@@ -400,7 +400,7 @@ Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partiti
 
 ### `Listeners[].default`
 
-Specifies a block of default configurations to apply to the gateway listener. All routes attached to the listener inherit the default configurations. You can specify override configurations that have precedence over default configurations in the [`override` block](#listeners-override) as well as in the `JWT` block in the [HTTP route configuration entry](/consul/docs/connect/config-entries/http-route).  
+Specifies a block of default configurations to apply to the gateway listener. All routes attached to the listener inherit the default configurations. You can specify override configurations that have precedence over default configurations in the [`override` block](#listeners-override) as well as in the `JWT` block in the [HTTP route configuration entry](/consul/docs/connect/config-entries/http-route).
 
 #### Values
 
@@ -409,7 +409,7 @@ Specifies a block of default configurations to apply to the gateway listener. Al
 
 ### `Listeners[].default{}.JWT`
 
-Specifies a block of default JWT verification configurations to apply to the gateway listener. Specify configurations that have precedence over the defaults in either the [`override.JWT` block](#listeners-override) or in the [`JWT` block](/consul/docs/connect/config-entries/http-route#rules-filters-jwt) in the HTTP route configuration. Refer to [Use JWTs to verify requests to API gateways](/consul/docs/connect/gateways/api-gateway/secure-traffic/verify-jwts-vms) for order of precedence and other details about using JWT verification in API gateways. 
+Specifies a block of default JWT verification configurations to apply to the gateway listener. Specify configurations that have precedence over the defaults in either the [`override.JWT` block](#listeners-override) or in the [`JWT` block](/consul/docs/connect/config-entries/http-route#rules-filters-jwt) in the HTTP route configuration. Refer to [Use JWTs to verify requests to API gateways](/consul/docs/connect/gateways/api-gateway/secure-traffic/verify-jwts-vms) for order of precedence and other details about using JWT verification in API gateways.
 
 #### Values
 
@@ -418,7 +418,7 @@ Specifies a block of default JWT verification configurations to apply to the gat
 
 ### `Listeners[].default{}.JWT{}.Providers`
 
-Specifies a list of default JWT provider configurations to apply to the gateway listener. A provider configuration contains the name of the provider and claims. Specify configurations that have precedence over the defaults in either the [`override.JWT.Providers` block](#listeners-override-providers) or in the [`JWT` block](/consul/docs/connect/config-entries/http-route#rules-filters-jwt-providers) of the HTTP route configuration. Refer to [Use JWTs to verify requests to API gateways](/consul/docs/connect/gateways/api-gateway/secure-traffic/verify-jwts-vms) for order of precedence and other details about using JWT verification in API gateways. 
+Specifies a list of default JWT provider configurations to apply to the gateway listener. A provider configuration contains the name of the provider and claims. Specify configurations that have precedence over the defaults in either the [`override.JWT.Providers` block](#listeners-override-providers) or in the [`JWT` block](/consul/docs/connect/config-entries/http-route#rules-filters-jwt-providers) of the HTTP route configuration. Refer to [Use JWTs to verify requests to API gateways](/consul/docs/connect/gateways/api-gateway/secure-traffic/verify-jwts-vms) for order of precedence and other details about using JWT verification in API gateways.
 
 #### Values
 
@@ -432,7 +432,7 @@ The following table describes the parameters you can specify in a member of the
 | `Name` | Specifies the name of the provider. | String | None |
 | `VerifyClaims` | Specifies a list of paths and a value that define the claim that Consul verifies when it receives a request. The `VerifyClaims` map specifies the following settings: <ul><li>`Path`: Specifies a list of one or more registered or custom claims.</li><li>`Value`: Specifies the expected value of the claim.</li></ul> | Map | None |
 
-Refer to [Configure JWT verification settings](#configure-jwt-verification-settings) for an example configuration. 
+Refer to [Configure JWT verification settings](#configure-jwt-verification-settings) for an example configuration.
 
 ### `Listeners[].override`
 
@@ -454,7 +454,7 @@ Specifies a block of JWT verification configurations to apply to the gateway lis
 
 ### `Listeners[].override{}.JWT{}.Providers`
 
-Specifies a list of JWT provider configurations to apply to the gateway listener. A provider configuration contains the name of the provider and claims. The override settings have precedence over `Listeners[].defaults{}.JWT{}.Providers` as well as any listener-specific configuration. 
+Specifies a list of JWT provider configurations to apply to the gateway listener. A provider configuration contains the name of the provider and claims. The override settings have precedence over `Listeners[].defaults{}.JWT{}.Providers` as well as any listener-specific configuration.
 
 #### Values
 
@@ -468,7 +468,7 @@ The following table describes the parameters you can specify in a member of the
 | `Name` | Specifies the name of the provider. | String | None |
 | `VerifyClaims` | Specifies a list of paths and a value that define the claim that Consul verifies when it receives a request. The `VerifyClaims` map specifies the following settings: <ul><li>`Path`: Specifies a list of one or more registered or custom claims.</li><li>`Value`: Specifies the expected value of the claim.</li></ul> | Map | None |
 
-Refer to [Configure JWT verification settings](#configure-jwt-verification-settings) for an example configuration. 
+Refer to [Configure JWT verification settings](#configure-jwt-verification-settings) for an example configuration.
 
 ## Examples
 
@@ -530,7 +530,7 @@ Listeners = [
     {
       "name": "listener-one",
       "port": 9001,
-      "protocol": "http",      
+      "protocol": "http",
       "override": {
         "JWT": {
           "Providers": [{
@@ -559,4 +559,4 @@ Listeners = [
 ```
 
 </Tab>
-</Tabs>
+</Tabs>

website/content/docs/connect/config-entries/file-system-certificate.mdx

@@ -0,0 +1,127 @@
+---
+layout: docs
+page_title: File System Certificate Configuration Reference
+description: Learn how to configure a file system certificate bound to an API Gateway on VMs.
+---
+
+# File system certificate configuration reference
+
+This topic provides reference information for the gateway file system certificate
+configuration entry. For information about certificate configuration for Kubernetes environments, refer to [Gateway Resource Configuration](/consul/docs/connect/gateways/api-gateway/configuration/gateway).
+
+## Configuration model
+
+The following list outlines field hierarchy, language-specific data types, and
+requirements in a `file-system-certificate` configuration entry. Click on a property name
+to view additional details, including default values.
+
+- [`Kind`](#kind): string | must be `"file-system-certificate"`
+- [`Name`](#name): string | no default
+- [`Namespace`](#namespace): string | no default <EnterpriseAlert inline />
+- [`Partition`](#partition): string | no default <EnterpriseAlert inline />
+- [`Meta`](#meta): map | no default
+- [`Certificate`](#certificate): string | no default
+- [`PrivateKey`](#privatekey): string | no default
+
+## Complete configuration
+
+When every field is defined, a `file-system-certificate` configuration entry has the following form:
+
+<CodeTabs>
+
+```HCL
+Kind = "file-system-certificate"
+Name = "<name of certificate>"
+
+Meta = {
+	"<any key>" = "<any value>"
+}
+
+Certificate = "<filepath to public certificate>"
+PrivateKey = "<filepath to private key>"
+```
+
+```JSON
+{
+	"Kind": "file-system-certificate",
+	"Name": "<name of certificate>",
+	"Meta": {
+		"any key": "any value"
+	}
+	"Certificate": "<filepath to public certificate>",
+	"PrivateKey": "<filepath to private key>"
+}
+```
+
+</CodeTabs>
+
+## Specification
+
+### `Kind`
+
+Specifies the type of configuration entry to implement.
+
+#### Values
+
+- Default: none
+- This field is required.
+- Data type: string that must equal `"file-system-certificate"`
+
+### `Name`
+
+Specifies a name for the configuration entry. The name is metadata that you can
+use to reference the configuration entry when performing Consul operations, such
+as applying a configuration entry to a specific cluster.
+
+#### Values
+
+- Default: none
+- This field is required.
+- Data type: string
+
+### `Namespace` <EnterpriseAlert inline />
+
+Specifies the Enterprise [namespace](/consul/docs/enterprise/namespaces) to apply to the configuration entry.
+
+#### Values
+
+- Default: `"default"` in Enterprise
+- Data type: string
+
+### `Partition` <EnterpriseAlert inline />
+
+Specifies the Enterprise [admin partition](/consul/docs/enterprise/admin-partitions) to apply to the configuration entry.
+
+#### Values
+
+- Default: `"default"` in Enterprise
+- Data type: string
+
+### `Meta`
+
+Specifies an arbitrary set of key-value pairs to associate with the gateway.
+
+#### Values
+
+- Default: none
+- Data type: map containing one or more keys and string values.
+
+### `Certificate`
+
+Specifies the filepath to a public certificate to use for TLS. This filepath must be accessible to the API gateway proxy at runtime.
+
+#### Values
+
+- Default: none
+- This field is required.
+- Data type: string value of the filepath to a public certificate
+
+### `PrivateKey`
+
+Specifies the filepath to a private key to use for TLS. This filepath must be accessible to the API gateway proxy at runtime.
+
+#### Values
+
+- Default: none
+- This field is required.
+- Data type: string value of the filepath to a private key

website/content/docs/connect/gateways/api-gateway/configuration/index.mdx

@@ -7,7 +7,7 @@ description: >-
 
 # Consul API gateway configuration overview
 
-This topic provides an overview of the configuration items you can use to create API gateways, configure listeners, define routes, and apply additional resources that may be necessary to operate Consul API gateways in your environment. 
+This topic provides an overview of the configuration items you can use to create API gateways, configure listeners, define routes, and apply additional resources that may be necessary to operate Consul API gateways in your environment.
 
 ## Configurations for virtual machines
 
@@ -18,9 +18,10 @@ Apply the following configuration items if your network runs on virtual machines
 | [`api-gateway`](/consul/docs/connect/config-entries/api-gateway) | Defines the main infrastructure resource for declaring an API gateway and listeners on the gateway. | [Deploy API gateway listeners on virtual machines](/consul/docs/connect/gateways/api-gateway/deploy/listeners-vms) |
 | [`http-route`](/consul/docs/connect/config-entries/http-route) | Enables HTTP traffic to reach services in the mesh from a listener on the gateway.| <nobr>[Define routes on virtual machines](/consul/docs/connect/gateways/api-gateway/define-routes/routes-vms)</nobr> |
 | [`tcp-route`](/consul/docs/connect/config-entries/tcp-route) | Enables TCP traffic to reach services in the mesh from a listener on the gateway.| [Define routes on virtual machines](/consul/docs/connect/gateways/api-gateway/define-routes/routes-vms) |
-| <nobr>[`inline-certificate`](/consul/docs/connect/config-entries/inline-certificate)</nobr> | Provides gateway artificats with a CA certificate so that requests between the user and the gateway endpoint are encrypted. | [Encrypt API gateway traffic on virtual machines](/consul/docs/connect/gateways/api-gateway/secure-traffic/encrypt-vms) |
+| <nobr>[`file-system-certificate`](/consul/docs/connect/config-entries/file-system-certificate)</nobr> | Provides gateway with a CA certificate so that requests between the user and the gateway endpoint are encrypted. | [Encrypt API gateway traffic on virtual machines](/consul/docs/connect/gateways/api-gateway/secure-traffic/encrypt-vms) |
+| <nobr>[`inline-certificate`](/consul/docs/connect/config-entries/inline-certificate)</nobr> | Provides gateway with a CA certificate so that requests between the user and the gateway endpoint are encrypted. | [Encrypt API gateway traffic on virtual machines](/consul/docs/connect/gateways/api-gateway/secure-traffic/encrypt-vms) |
 | [`service-intentions`](/consul/docs/connect/config-entries/service-intentions) | Specifies traffic communication rules between services in the mesh. Intentions also enforce rules for service-to-service traffic routed through a Consul API gateway. | General configuration for securing a service mesh |
- 
+
 ## Configurations for Kubernetes
 
 Apply the following configuration items if your network runs on Kubernetes:
@@ -32,7 +33,7 @@ Apply the following configuration items if your network runs on Kubernetes:
 | [`GatewayClassConfig`](/consul/docs/connect/gateways/api-gateway/configuration/gatewayclassconfig) | Describes additional gateway-related configuration parameters for the `GatewayClass` resource. | [Deploy listeners on Kubernetes](/consul/docs/connect/gateways/api-gateway/deploy/listeners-k8s) |
 | [`Routes`](/consul/docs/connect/gateways/api-gateway/configuration/routes) | Specifies paths from the gateway listener to backend services. | <nobr>[Define routes on Kubernetes](/consul/docs/connect/gateways/api-gateway/define-routes/routes-k8s)</nobr><p>[Reroute traffic in Kubernetes](/consul/docs/connect/gateways/api-gateway/define-routes/reroute-http-requests)</p><p>[Route traffic to peered services in Kubernetes](/consul/docs/connect/gateways/api-gateway/define-routes/route-to-peered-services)</p> |
 | [`MeshServices`](/consul/docs/connect/gateways/api-gateway/configuration/meshservices) | Enables routes to reference services in Consul. | [Route traffic to peered services in Kubernetes](/consul/docs/connect/gateways/api-gateway/define-routes/route-to-peered-services) |
-| [`ServiceIntentions`](/consul/docs/connect/config-entries/service-intentions) | Specifies traffic communication rules between services in the mesh. Intentions also enforce rules for service-to-service traffic routed through a Consul API gateway. | General configuration for securing a service mesh | 
+| [`ServiceIntentions`](/consul/docs/connect/config-entries/service-intentions) | Specifies traffic communication rules between services in the mesh. Intentions also enforce rules for service-to-service traffic routed through a Consul API gateway. | General configuration for securing a service mesh |
 
 <!-- Reuse later for a topic about creating custom api gateway classes
 You can create a basic Gateway object using the default [`gatewayClassName`](/consul/docs/connect/gateways/api-gateway/configuration/gateway#gatewayclassname) (`consul`). If you want to create custom Gateways suitable for your environment, complete the following steps:

website/content/docs/connect/gateways/api-gateway/configuration/routes.mdx

@@ -300,10 +300,10 @@ The following example creates a route named `example-route` in namespace `gatewa
     - group: gateway.networking.k8s.io
       kind: HTTPRoute
       namespace: gateway-namespace
-      to:
-        - group: ""
-          kind: Service
-          name: echo
+    to:
+      - group: ""
+        kind: Service
+        name: echo
   ```
 
 </CodeBlockConfig>

website/content/docs/dynamic-app-config/kv/store.mdx

@@ -0,0 +1,89 @@
+---
+layout: docs
+page_title: Store and access key/value data
+description: >-
+  Consul includes a key/value store for that you can use to dynamically configure apps. Learn how to add and manage data in the Consul KV store using the Consul
+  command-line interface.
+---
+
+# Store and access key/value data
+
+This page describes the processes for interacting with Consul's KV store. You can interact with the KV store using the [`consul kv` CLI command](/consul/commands/kv) or the [`/kv` endpoint](/consul/api-docs/kv).
+
+## Add data to KV store
+
+To insert values into the KV store or update an existing value, use the `consul kv put` command. The first entry after the command is the key and the second entry is the value.
+
+```shell-session
+$ consul kv put redis/config/minconns 1
+Success! Data written to: redis/config/minconns
+```
+
+In the following example, the key is `redis/config/maxconns` and the value is set to `25`.
+
+```shell-session
+$ consul kv put redis/config/maxconns 25
+Success! Data written to: redis/config/maxconns
+```
+
+In the following example, the command includes a `flags` value of 42. Keys support setting a 64-bit integer flag value that is not used internally by Consul but can be used by clients to add metadata to a KV pair.
+
+```shell-session
+$ consul kv put -flags=42 redis/config/users/admin zaphod
+Success! Data written to: redis/config/users/admin
+```
+
+## Query data from KV store
+
+To query for the value of one of the keys in the KV store, use the `consul kv get` command.
+
+```shell-session
+$ consul kv get redis/config/minconns
+1
+```
+
+To retrieve metadata you included as `flags`, using the `-detailed` command line flag.
+
+```shell-session
+$ consul kv get -detailed redis/config/users/admin
+```
+
+```plaintext hideClipboard
+CreateIndex      14
+Flags            42
+Key              redis/config/users/admin
+LockIndex        0
+ModifyIndex      14
+Session          -
+Value            zaphod
+```
+
+To list all the keys in the store, use the `recurse` option. Results return in lexicographical order.
+
+```shell-session
+$ consul kv get -recurse
+```
+
+```plaintext hideClipboard
+redis/config/maxconns:25
+redis/config/minconns:1
+redis/config/users/admin:zaphod
+```
+
+## Delete data
+
+To delete the value of one of the keys in the KV store, use the `consul kv delete` command.
+
+```shell-session
+$ consul kv delete redis/config/minconns
+Success! Deleted key: redis/config/minconns
+```
+
+Although the keys in the KV store are stored in a flat structure, you can manipulate keys that share a prefix as a group, as if they were in folders or subfolders, by using the `-recurse` flag.
+
+The following example deletes all the keys with the `redis` prefix using the `-recurse` option.
+
+```shell-session
+$ consul kv delete -recurse redis
+Success! Deleted keys with prefix: redis
+```

website/content/docs/install/performance.mdx

@@ -110,9 +110,9 @@ Here are some general recommendations:
 - For DNS-heavy workloads, configuring all Consul agents in a cluster with the
   [`allow_stale`](/consul/docs/agent/config/config-files#allow_stale) configuration option will allow reads to
   scale across all Consul servers, not just the leader. Consul 0.7 and later enables stale reads
-  for DNS by default. See [Stale Reads](/consul/tutorials/networking/dns-caching#stale-reads) in the
-  [DNS Caching](/consul/tutorials/networking/dns-caching) guide for more details. It's also good to set
-  reasonable, non-zero [DNS TTL values](/consul/tutorials/networking/dns-caching#ttl-values) if your clients will
+  for DNS by default. See [Stale Reads](/consul/docs/services/discovery/dns-cache#stale-reads) in the
+  [DNS Caching](/consul/docs/services/discovery/dns-cache) guide for more details. It's also good to set
+  reasonable, non-zero [DNS TTL values](/consul/docs/services/discovery/dns-cache#ttl-values) if your clients will
   respect them.
 
 - In other applications that perform high volumes of reads against Consul, consider using the

website/content/docs/nia/api/tasks.mdx

@@ -400,5 +400,5 @@ The task object is used by the Task APIs as part of a request or response. It re
 | `providers`             | list[string] | Optional | The list of provider names that the task's module uses.                                                                                                                                                                                                                               | none                                                         |
 | `variables`             | map[string]  | Optional | The map of variables that are provided to the task's module.                                                                                                                                                                                                                          | none                                                         |
 | `version`               | string       | Optional | The version of the configured module that the task uses.                                                                                                                                                                                                                              | The latest version.                                          |
-| `terraform_version`     | string       | Optional | <EnterpriseAlert inline /> **Deprecated in CTS 0.6.0 and will be removed in 0.8.0. Review `terraform_version` in `terraform_cloud_workspace` instead.** The version of Terraform to use for the Terraform Cloud workspace associated with the task. This is only available when used with the [Terraform Cloud driver](/consul/docs/nia/configuration#terraform-cloud-driver).                                                  | The latest compatible version supported by the organization. |
-| `terraform_cloud_workspace` | object | Optional | <EnterpriseAlert inline /> The [configurable attributes of the Terraform Cloud workspace](/consul/docs/nia/configuration#terraform_cloud_workspace) associated with the task. This option is only available when used with the [Terraform Cloud driver](/consul/docs/nia/configuration#terraform-cloud-driver).| none |
+| `terraform_version`     | string       | Optional | <EnterpriseAlert inline /> **Deprecated in CTS 0.6.0 and will be removed in 0.8.0. Review `terraform_version` in `terraform_cloud_workspace` instead.** The version of Terraform to use for the HCP Terraform workspace associated with the task. This is only available when used with the [HCP Terraform driver](/consul/docs/nia/configuration#hcp-terraform-driver).                                                  | The latest compatible version supported by the organization. |
+| `terraform_cloud_workspace` | object | Optional | <EnterpriseAlert inline /> The [configurable attributes of the HCP Terraform workspace](/consul/docs/nia/configuration#terraform_cloud_workspace) associated with the task. This option is only available when used with the [HCP Terraform driver](/consul/docs/nia/configuration#hcp-terraform-driver).| none |

website/content/docs/nia/architecture.mdx

@@ -41,7 +41,7 @@ A driver encapsulates the resources required to communicate the updates to the
 network infrastructure. The following [drivers](/consul/docs/nia/network-drivers#terraform) are supported:
 
 - Terraform driver
-- Terraform Cloud driver<EnterpriseAlert inline />
+- HCP Terraform driver<EnterpriseAlert inline />
 
 Each driver includes a set of providers that [enables support](/consul/docs/nia/terraform-modules) for a wide variety of infrastructure applications.
 

website/content/docs/nia/compatibility.mdx

@@ -24,7 +24,7 @@ Below are CTS versions with supported Consul versions. The latest CTS binary sup
 
 CTS integration with Terraform is supported for the following:
 
-| CTS Version         | Terraform CLI Version | Terraform Cloud Version | Terraform Enterprise Version |
+| CTS Version         | Terraform CLI Version | HCP Terraform Version | Terraform Enterprise Version |
 | :------------------ | :-------------------- | :---------------------- | :--------------------------- |
 | CTS Enterprise 0.7   | 0.13-1.2             | Latest                  | v202010-2 - Latest           |
 | CTS Enterprise 0.4+ | 0.13 - 1.1            | Latest                  | v202010-2 - Latest           |

website/content/docs/nia/configuration.mdx

@@ -302,7 +302,7 @@ task {
 - `module` - (string: required) Module is the location the driver uses to discover the Terraform module used for automation. The module's source can be local or remote on the [Terraform Registry](https://registry.terraform.io/) or private module registry. Read more on [Terraform module source and other supported types here](/terraform/language/modules/sources).
 
   - To use a private module with the [`terraform` driver](#terraform-driver), run the command [`terraform login [hostname]`](/terraform/tutorials/cloud/cloud-login?utm_source=docs) to authenticate the local Terraform CLI prior to starting CTS.
-  - To use a private module with the [`terraform_cloud` driver](#terraform-cloud-driver), no extra steps are needed.
+  - To use a private module with the [`terraform_cloud` driver](#hcp-terraform-driver), no extra steps are needed.
 
     ```hcl
     // local module example: "./terraform-cts-hello"
@@ -339,12 +339,12 @@ task {
 - `condition` - (obj: required) The requirement that, when met, triggers CTS to execute the task. Only one `condition` may be configured per task. CTS supports different types of conditions, which each have their own configuration options. See [Task Condition](#task-condition) configuration for full details on configuration options for each condition type.
 - `source_input` - (obj) **Deprecated in CTS 0.5.0 and will be removed in 0.8.0. See the `module_input` block instead.**
 - `module_input` - (obj) Specifies a Consul object containing values or metadata to be provided to the Terraform Module. The `module_input` block defines any extra module inputs needed for task execution.  This is in addition to any module input provided by the `condition` block or `services` field (deprecated). Multiple `module_input` blocks can be configured per task. [Task Module Input](#task-module-input) configuration for full details on usage and restrictions.
-- `terraform_version` - (string) <EnterpriseAlert inline />  **Deprecated in CTS 0.6.0 and will be removed in 0.8.0. Review `terraform_cloud_workspace.terraform_version` instead.** The version of Terraform to use for the Terraform Cloud workspace associated with the task. Defaults to the latest compatible version supported by the organization. This option is only available when used with the [Terraform Cloud driver](#terraform-cloud-driver); otherwise, set the version within the [Terraform driver](#terraform-driver).
-- `terraform_cloud_workspace` - (obj) <EnterpriseAlert inline /> Configures attributes of the Terraform Cloud workspace associated with the task. This option is only available when used with the [Terraform Cloud driver](#terraform-cloud-driver). For global configurations of all workspaces, review [`driver.workspaces`](#workspaces).
-  - `execution_mode` - (string: "remote") The execution mode that determines whether to use Terraform Cloud as the Terraform execution platform. Only supports "remote" or "agent".
+- `terraform_version` - (string) <EnterpriseAlert inline />  **Deprecated in CTS 0.6.0 and will be removed in 0.8.0. Review `terraform_cloud_workspace.terraform_version` instead.** The version of Terraform to use for the HCP Terraform workspace associated with the task. Defaults to the latest compatible version supported by the organization. This option is only available when used with the [HCP Terraform driver](#hcp-terraform-driver); otherwise, set the version within the [Terraform driver](#terraform-driver).
+- `terraform_cloud_workspace` - (obj) <EnterpriseAlert inline /> Configures attributes of the HCP Terraform workspace associated with the task. This option is only available when used with the [HCP Terraform driver](#hcp-terraform-driver). For global configurations of all workspaces, review [`driver.workspaces`](#workspaces).
+  - `execution_mode` - (string: "remote") The execution mode that determines whether to use HCP Terraform as the Terraform execution platform. Only supports "remote" or "agent".
   - `agent_pool_id` - (string) Only supported if `execution_mode` is set to "agent". The ID of the agent pool that should run the Terraform workloads. Either `agent_pool_id` or `agent_pool_name` are required if `execution_mode` is set to "agent". `agent_pool_id` takes precedence over `agent_pool_name` if both are provided.
   - `agent_pool_name` - (string) Only supported if `execution_mode` is set to "agent". The name of the agent pool that should run the Terraform workloads. Only supported if `execution_mode` is set to "agent". Either `agent_pool_id` or `agent_pool_name` are required. `agent_pool_id` takes precedence over `agent_pool_name` if both are provided.
-  - `terraform_version` - (string) The version of Terraform to use for the Terraform Cloud workspace associated with the task. Defaults to the latest compatible version supported by the organization.
+  - `terraform_version` - (string) The version of Terraform to use for the HCP Terraform workspace associated with the task. Defaults to the latest compatible version supported by the organization.
 
 ### Task Condition
 
@@ -653,7 +653,7 @@ driver "terraform" {
 - `backend` - (obj) The backend stores [Terraform state files](/terraform/language/state) for each task. This option is similar to the [Terraform backend configuration](/terraform/language/settings/backends/configuration). CTS supports Terraform backends used as a state store.
   - Supported backend options: [azurerm](/terraform/language/settings/backends/azurerm), [consul](/terraform/language/settings/backends/consul), [cos](/terraform/language/settings/backends/cos), [gcs](/terraform/language/settings/backends/gcs), [kubernetes](/terraform/language/settings/backends/kubernetes), [local](/terraform/language/settings/backends/local), [manta](/terraform/language/v1.2.x/settings/backends/manta), [pg](/terraform/language/settings/backends/pg) (Terraform v0.14+), [s3](/terraform/language/settings/backends/s3). Visit the Terraform documentation links for details on backend configuration options.
   - If omitted, CTS will generate default values and use configurations from the [`consul` block](#consul) to configure [Consul as the backend](/terraform/language/settings/backends/consul), which stores Terraform statefiles in the Consul KV. The [ACL token provided for Consul authentication](#consul) is used to read and write to the KV store and requires [Consul KV privileges](/consul/tutorials/network-infrastructure-automation/consul-terraform-sync-secure?utm_source=docs#configure-acl-privileges-for-consul-terraform-sync). The Consul KV path is the base path to store state files for tasks. The full path of each state file will have the task identifier appended to the end of the path, e.g. `consul-terraform-sync/terraform-env:task-name`.
-  - The remote enhanced backend is not supported with the Terraform driver to run operations in Terraform Cloud. Use the [Terraform Cloud driver](#terraform-cloud-driver) to integrate CTS with Terraform Cloud for remote workspaces and remote operations.
+  - The remote enhanced backend is not supported with the Terraform driver to run operations in HCP Terraform. Use the [HCP Terraform driver](#hcp-terraform-driver) to integrate CTS with HCP Terraform for remote workspaces and remote operations.
   - The `local` backend type is not supported with CTS instances configured for high availability. If high availability is configured and the Terraform backend type is `local`, CTS logs an error and exits.
 - `log` - (bool) Enable all Terraform output (stderr and stdout) to be included in the CTS log. This is useful for debugging and development purposes. It may be difficult to work with log aggregators that expect uniform log format.
 - `path` - (string) The file path to install Terraform or discover an existing Terraform binary. If omitted, Terraform will be installed in the same directory as the CTS daemon. To resolve an incompatible Terraform version or to change versions will require removing the existing binary or change to a different path.
@@ -661,7 +661,7 @@ driver "terraform" {
 - `required_providers` - (obj: required) Declare each Terraform provider used across all tasks. This can be configured the same as how you would configure [Terraform `terraform.required_providers`](/terraform/language/providers/requirements#requiring-providers) field to specify the source and version for each provider. CTS will process these requirements when preparing each task that uses the provider.
 - `version` - (string) The Terraform version to install and run in automation for task execution. If omitted, the driver will install the latest [compatible release of Terraform](/consul/docs/nia/compatibility#terraform). To change versions, remove the existing binary or change the path to install the desired version. Verify that the desired Terraform version is compatible across all Terraform modules used for CTS automation.
 
-## Terraform Cloud Driver
+## HCP Terraform Driver
 
 <EnterpriseAlert>
   This feature requires{' '}
@@ -671,9 +671,9 @@ driver "terraform" {
   which is available with <strong>Consul Enterprise</strong>.
 </EnterpriseAlert>
 
-The Terraform Cloud driver enables CTS Enterprise to integrate with **Terraform Cloud**, including both the [self-hosted distribution](https://www.hashicorp.com/products/terraform/editions/enterprise) and the [managed service](https://www.hashicorp.com/products/terraform/editions/cloud). With this driver, CTS automates Terraform runs and remote operations for workspaces.
+The HCP Terraform driver enables CTS Enterprise to integrate with HCP Terraform, including both the [self-hosted distribution](https://www.hashicorp.com/products/terraform/editions/enterprise) and the [managed service](https://www.hashicorp.com/products/terraform/editions/cloud). With this driver, CTS automates Terraform runs and remote operations for workspaces.
 
-An overview of features enabled with Terraform Cloud can be viewed within the [Network Drivers](/consul/docs/nia/network-drivers) documentation.
+An overview of features enabled with HCP Terraform can be viewed within the [Network Drivers](/consul/docs/nia/network-drivers) documentation.
 
 Only one network driver can be configured per deployment of CTS.
 
@@ -701,16 +701,16 @@ driver "terraform-cloud" {
 }
 ```
 
-- `hostname` - (string) The Terraform Cloud hostname to connect to. Can be overridden with the `TFC_HOSTNAME` environment variable.
-- `organization` - (string) The Terraform Cloud organization that hosts the managed workspaces by CTS. Can be overridden with the `TFC_ORGANIZATION` environment variable.
-- `token` - (string) Required [Team API token](/terraform/cloud-docs/users-teams-organizations/api-tokens#team-api-tokens) used for authentication with Terraform Cloud and workspace management. Only workspace permissions are needed for CTS. The token can also be provided using the `TFC_TOKEN` environment variable.
-  - We recommend creating a dedicated team and team API token to isolate automation by CTS from other Terraform Cloud operations.
+- `hostname` - (string) The HCP Terraform hostname to connect to. Can be overridden with the `TFC_HOSTNAME` environment variable.
+- `organization` - (string) The HCP Terraform organization that hosts the managed workspaces by CTS. Can be overridden with the `TFC_ORGANIZATION` environment variable.
+- `token` - (string) Required [Team API token](/terraform/cloud-docs/users-teams-organizations/api-tokens#team-api-tokens) used for authentication with HCP Terraform and workspace management. Only workspace permissions are needed for CTS. The token can also be provided using the `TFC_TOKEN` environment variable.
+  - We recommend creating a dedicated team and team API token to isolate automation by CTS from other HCP Terraform operations.
 - `workspace_prefix` - (string) **Deprecated in CTS 0.5.0**, use the [`workspaces.prefix`](#prefix) option instead. Specifies a prefix to prepend to the automatically-generated workspace names used for automation. This prefix will be used by all tasks that use this driver. By default, when no prefix is configured, the workspace name will be the task name. When a prefix is configured, the workspace name will be `<workspace_prefix value>-<task name>`, with the character '-' between the workspace prefix and task name. For example, if you configure the prefix as "cts", then a task with the name "task-firewall" will have the workspace name "cts-task-firewall".
-- `workspaces` - Configure CTS management of Terraform Cloud workspaces.
+- `workspaces` - Configure CTS management of HCP Terraform workspaces.
   - `prefix` - (string) Specifies a prefix to prepend to the workspace names used for CTS task automation. This prefix will be used by all tasks that use this driver. By default, when no prefix is configured, the workspace name will be the task name. When a prefix is configured, the workspace name will be `<prefix><task name>`. For example, if you configure the prefix as "cts_", then a task with the name "task_firewall" will have the workspace name "cts_task_firewall".
-  - `tags` - (list[string]) Tags for CTS to add to all automated workspaces when the workspace is first created or discovered. Tags are added to discovered workspaces only if the workspace meets [automation requirements](/consul/docs/nia/network-drivers/terraform-cloud#remote-workspaces) and satisfies the allowlist and denylist tag options. This option will not affect existing tags. Tags that were manually removed during runtime will be re-tagged when CTS restarts. Compatible with Terraform Cloud and Terraform Enterprise v202108-1+
-  - `tags_allowlist` - (list[string]) Tag requirement to use as a provision check for CTS automation of workspaces. When configured, Terraform Cloud workspaces must have at least one tag from the allow list for CTS to automate the workspace and runs. Compatible with Terraform Cloud and Terraform Enterprise v202108-1+.
-  - `tags_denylist` - (list[string]) Tag restriction to use as a provision check for CTS automation of workspaces. When configured, Terraform Cloud workspaces must not have any tag from the deny list for CTS to automate the workspace and runs. Denied tags have higher priority than tags set in the `tags_allowlist` option. Compatible with Terraform Cloud and Terraform Enterprise v202108-1+.
+  - `tags` - (list[string]) Tags for CTS to add to all automated workspaces when the workspace is first created or discovered. Tags are added to discovered workspaces only if the workspace meets [automation requirements](/consul/docs/nia/network-drivers/hcp-terraform#remote-workspaces) and satisfies the allowlist and denylist tag options. This option will not affect existing tags. Tags that were manually removed during runtime will be re-tagged when CTS restarts. Compatible with HCP Terraform and Terraform Enterprise v202108-1+
+  - `tags_allowlist` - (list[string]) Tag requirement to use as a provision check for CTS automation of workspaces. When configured, HCP Terraform workspaces must have at least one tag from the allow list for CTS to automate the workspace and runs. Compatible with HCP Terraform and Terraform Enterprise v202108-1+.
+  - `tags_denylist` - (list[string]) Tag restriction to use as a provision check for CTS automation of workspaces. When configured, HCP Terraform workspaces must not have any tag from the deny list for CTS to automate the workspace and runs. Denied tags have higher priority than tags set in the `tags_allowlist` option. Compatible with HCP Terraform and Terraform Enterprise v202108-1+.
 - `required_providers` - (obj: required) Declare each Terraform provider used across all tasks. This can be configured the same as how you would configure [Terraform `terraform.required_providers`](/terraform/language/providers/requirements#requiring-providers) field to specify the source and version for each provider. CTS will process these requirements when preparing each task that uses the provider.
 - `tls` - Configure TLS to allow HTTPS connections to [Terraform Enterprise](/terraform/enterprise/install/interactive/installer#tls-key-amp-cert).
   - `enabled` - (bool) Enable TLS. Providing a value for any of the TLS options will enable this parameter implicitly.
@@ -727,7 +727,7 @@ driver "terraform-cloud" {
       }
       ```
 
-CTS generates local artifacts to prepare configuration versions used for workspace runs. The location of the files created can be set with the [`working_dir`](/consul/docs/nia/configuration#working_dir) option or configured per task. When a task is configured with a local module and is run with the Terraform Cloud driver, the local module is copied and uploaded as a part of the configuration version.
+CTS generates local artifacts to prepare configuration versions used for workspace runs. The location of the files created can be set with the [`working_dir`](/consul/docs/nia/configuration#working_dir) option or configured per task. When a task is configured with a local module and is run with the HCP Terraform driver, the local module is copied and uploaded as a part of the configuration version.
 
 The version of Terraform to use for each workspace can also be set within the [task](#task) configuration.
 

website/content/docs/nia/enterprise/index.mdx

@@ -9,22 +9,22 @@ description: >-
 
 Consul-Terraform-Sync (CTS) Enterprise is available with [Consul Enterprise](https://www.hashicorp.com/products/consul) and requires a Consul [license](/consul/docs/nia/enterprise/license) to be applied.
 
-Enterprise features of CTS address organization complexities of collaboration, operations, scale, and governance. CTS Enterprise supports an official integration with [Terraform Cloud](https://cloud.hashicorp.com/products/terraform) and [Terraform Enterprise](/terraform/enterprise), the self-hosted distribution, to extend insight into dynamic updates of your network infrastructure.
+Enterprise features of CTS address organization complexities of collaboration, operations, scale, and governance. CTS Enterprise supports an official integration with [HCP Terraform](https://cloud.hashicorp.com/products/terraform) and [Terraform Enterprise](/terraform/enterprise), the self-hosted distribution, to extend insight into dynamic updates of your network infrastructure.
 
 | Features | Community Edition | Enterprise |
 |----------|-------------|------------|
 | Consul Namespace | Default namespace only | Filter task triggers by any namespace |
-| Automation Driver | Terraform Community Edition | Terraform Community Edition, Terraform Cloud, or Terraform Enterprise |
-| Terraform Workspaces | Local | Local workspaces with the Terraform driver or [remote workspaces](/terraform/cloud-docs/workspaces) with the Terraform Cloud driver |
-| Terraform Backend Options | [azurerm](/terraform/language/settings/backends/azurerm), [consul](/terraform/language/settings/backends/consul), [cos](/terraform/language/settings/backends/cos), [gcs](/terraform/language/settings/backends/gcs), [kubernetes](/terraform/language/settings/backends/kubernetes), [local](/terraform/language/settings/backends/local), [manta](/terraform/language/v1.2.x/settings/backends/manta), [pg](/terraform/language/settings/backends/pg), and [s3](/terraform/language/settings/backends/s3) with the Terraform driver | The supported backends for CTS with the Terraform driver or Terraform Cloud with the Terraform Cloud driver |
-| Terraform Version | One Terraform version for all tasks | Optional Terraform version per task when using the Terraform Cloud driver |
-| Terraform Run Output | CTS logs | CTS logs or Terraform output organized by Terraform Cloud remote workspaces |
+| Automation Driver | Terraform Community Edition | Terraform Community Edition, HCP Terraform, or Terraform Enterprise |
+| Terraform Workspaces | Local | Local workspaces with the Terraform driver or [remote workspaces](/terraform/cloud-docs/workspaces) with the HCP Terraform driver |
+| Terraform Backend Options | [azurerm](/terraform/language/settings/backends/azurerm), [consul](/terraform/language/settings/backends/consul), [cos](/terraform/language/settings/backends/cos), [gcs](/terraform/language/settings/backends/gcs), [kubernetes](/terraform/language/settings/backends/kubernetes), [local](/terraform/language/settings/backends/local), [manta](/terraform/language/v1.2.x/settings/backends/manta), [pg](/terraform/language/settings/backends/pg), and [s3](/terraform/language/settings/backends/s3) with the Terraform driver | The supported backends for CTS with the Terraform driver or HCP Terraform with the HCP Terraform driver |
+| Terraform Version | One Terraform version for all tasks | Optional Terraform version per task when using the HCP Terraform driver |
+| Terraform Run Output | CTS logs | CTS logs or Terraform output organized by HCP Terraform remote workspaces |
 | Credentials and secrets | On disk as `.tfvars` files or in shell environment | Secured variables stored in remote workspace |
-| Audit | | Terraform audit logs ([Terraform Cloud](/terraform/cloud-docs/api-docs/audit-trails) or [Terraform Enterprise](/terraform/enterprise/admin/infrastructure/logging)) |
-| Collaboration | | Run [history](/terraform/cloud-docs/run/manage), [triggers](/terraform/cloud-docs/workspaces/settings/run-triggers), and [notifications](/terraform/cloud-docs/workspaces/settings/notifications) supported on Terraform Cloud |
+| Audit | | Terraform audit logs ([HCP Terraform](/terraform/cloud-docs/api-docs/audit-trails) or [Terraform Enterprise](/terraform/enterprise/admin/infrastructure/logging)) |
+| Collaboration | | Run [history](/terraform/cloud-docs/run/manage), [triggers](/terraform/cloud-docs/workspaces/settings/run-triggers), and [notifications](/terraform/cloud-docs/workspaces/settings/notifications) supported on HCP Terraform |
 | Governance | | [Sentinel](/terraform/cloud-docs/policy-enforcement) to enforce governance policies as code |
 
-The [Terraform Cloud driver](/consul/docs/nia/configuration#terraform-cloud-driver) enables CTS Enterprise to integrate with Terraform Cloud or Terraform Enterprise. The [Terraform Cloud driver](/consul/docs/nia/network-drivers/terraform-cloud) page provides an overview of how the integration works within CTS.
+The [HCP Terraform driver](/consul/docs/nia/configuration#terraform-cloud-driver) enables CTS Enterprise to integrate with HCP Terraform or Terraform Enterprise. The [HCP Terraform driver](/consul/docs/nia/network-drivers/terraform-cloud) page provides an overview of how the integration works within CTS.
 
 ## Consul Admin Partition Support
 CTS subscribes to a Consul agent. Depending on the admin partition the Consul agent is a part of and the services within the admin partition, CTS will be able to subscribe to those services and support the automation workflow. As such, admin partitions are not relevant to the CTS workflow. We recommend deploying a single CTS instance that subscribes to services/KV within a single partition and using a different CTS instance (or instances) to subscribe to services/KV in another partition.

website/content/docs/nia/index.mdx

@@ -53,7 +53,7 @@ CTS is available as an open source and enterprise distribution. Follow the [Auto
 
 - `Tasks` - A task is the translation of dynamic service information from the Consul Catalog into network infrastructure changes downstream.
 
-- `Terraform Cloud` - Per the [Terraform documentation](/terraform/cloud-docs), "Terraform Cloud" describes both Terraform Cloud and Terraform Enterprise, which are different distributions of the same application. Documentation will apply to both distributions unless specifically stated otherwise.
+- `HCP Terraform` - Per the [Terraform documentation](/terraform/cloud-docs), "HCP Terraform" describes both HCP Terraform and Terraform Enterprise, which are different distributions of the same application. Documentation will apply to both distributions unless specifically stated otherwise.
 
 - `Terraform Module` - A [Terraform module](/terraform/language/modules) is a container for multiple Terraform resources that are used together.
 

website/content/docs/nia/network-drivers/hcp-terraform.mdx

@@ -1,31 +1,31 @@
 ---
 layout: docs
-page_title: Terraform Cloud Driver
+page_title: HCP Terraform Driver
 description: >-
-  Consul-Terraform-Sync Network Drivers with Terraform Cloud
+  Consul-Terraform-Sync Network Drivers with HCP Terraform
 ---
 
-# Terraform Cloud Driver
+# HCP Terraform Driver
 <EnterpriseAlert>
   This feature requires{' '}
   <a href="https://www.hashicorp.com/products/consul/features">Consul-Terraform-Sync Enterprise</a>{' '}
   which is available with <strong>Consul Enterprise</strong>.
 </EnterpriseAlert>
 
-Consul-Terraform-Sync (CTS) is more powerful when you integrate it with [Terraform Cloud](https://cloud.hashicorp.com/products/terraform). Integrating with Terraform Cloud provides features, such as enhanced workspaces and insight into Terraform operations as CTS dynamically updates your network infrastructure. CTS is compatible with both the [self-hosted](https://www.hashicorp.com/products/terraform/editions/enterprise) and [managed service](https://www.hashicorp.com/products/terraform/editions/cloud) versions of Terraform Cloud. It also supports all [tiers](https://www.hashicorp.com/products/terraform/pricing) of the Terraform Cloud managed service.
+Consul-Terraform-Sync (CTS) is more powerful when you integrate it with [HCP Terraform](https://cloud.hashicorp.com/products/terraform). Integrating with HCP Terraform provides features, such as enhanced workspaces and insight into Terraform operations as CTS dynamically updates your network infrastructure. CTS is compatible with both the [self-hosted](https://www.hashicorp.com/products/terraform/editions/enterprise) and [managed service](https://www.hashicorp.com/products/terraform/editions/cloud) versions of HCP Terraform. It also supports all [tiers](https://www.hashicorp.com/products/terraform/pricing) of the HCP Terraform managed service.
 
-This page describes how the Terraform Cloud driver operates within CTS.
+This page describes how the HCP Terraform driver operates within CTS.
 
 ## Terraform Workspace Automation
 
-CTS manages Terraform runs following the [API-driven run workflow](/terraform/cloud-docs/run/api) for workspaces in Terraform Cloud.
+CTS manages Terraform runs following the [API-driven run workflow](/terraform/cloud-docs/run/api) for workspaces in HCP Terraform.
 
 On startup, CTS:
-1. Creates or discovers Terraform Cloud workspaces corresponding to the configured tasks.
+1. Creates or discovers HCP Terraform workspaces corresponding to the configured tasks.
 2. Prepares the local environment and generates Terraform configuration files that make up the root module for each task.
-3. Packages the generated files and uploads them as a configuration version for the task's workspace on Terraform Cloud.
+3. Packages the generated files and uploads them as a configuration version for the task's workspace on HCP Terraform.
 
-Once all workspaces are set up, CTS monitors the Consul catalog for service changes. When relevant changes are detected, the Terraform Cloud driver dynamically updates input variables for that task directly as [workspace variables](/terraform/cloud-docs/workspaces/variables) using the Terraform Cloud API. The driver then queues a run on the workspace, with auto-apply enabled, to update your network infrastructure.
+Once all workspaces are set up, CTS monitors the Consul catalog for service changes. When relevant changes are detected, the HCP Terraform driver dynamically updates input variables for that task directly as [workspace variables](/terraform/cloud-docs/workspaces/variables) using the HCP Terraform API. The driver then queues a run on the workspace, with auto-apply enabled, to update your network infrastructure.
 
 ~> **Note:** Although workspaces for tasks are executed in isolated environments, this does not guarantee the infrastructure changes from concurrent task executions are independent. Ensure that modules across all tasks are not modifying the same resource objects or have overlapping changes that may result in race conditions during automation.
 
@@ -61,8 +61,8 @@ CTS can automate remote workspaces with either auto apply or manual apply config
 When CTS detects new changes for a workspace that already has a run pending on approval, CTS will discard the stale run and queue a new run with the latest values. The new run will go through plan and then again wait on an operator to approve it. Only once the run is approved will the infrastructure be updated with the latest Consul changes.
 
 There are two approaches to setup manual apply for a workspace managed by CTS based on how the workspace is created.
-* For CTS created workspaces, update the apply method from auto to manual via the Terraform Cloud web application or API.
-* For pre-configured workspaces, create the workspace prior to CTS task automation via the Terraform Cloud web application or API.
+* For CTS created workspaces, update the apply method from auto to manual via the HCP Terraform web application or API.
+* For pre-configured workspaces, create the workspace prior to CTS task automation via the HCP Terraform web application or API.
   1. Create a workspace with the same name as the desired task.
   1. Set the workspace to [API-driven run workflow](/terraform/cloud-docs/run/api) and the execution mode to remote.
   1. Ensure that the apply method for the workspace is set to manual apply.
@@ -72,7 +72,7 @@ There are two approaches to setup manual apply for a workspace managed by CTS ba
 
 ## Configuration Version
 
-An example configuration version for a task named "cts-example" would have the folder structure below when running with the Terraform Cloud driver and using the default working directory.
+An example configuration version for a task named "cts-example" would have the folder structure below when running with the HCP Terraform driver and using the default working directory.
 
 ```shell-session
 $ tree sync-tasks/
@@ -103,23 +103,23 @@ You can view the latest service information in the Terraform UI by navigating to
 
 ~> **Caution:** Dynamic variables maintained by CTS are formatted for automation. Unexpected manual changes to these variables may result in automation errors.
 
-## Setting Up Terraform Cloud Driver
+## Setting Up HCP Terraform Driver
 
 ### Deployment
 
-Because a CTS instance can only be configured with one driver, an instance can only be associated with either a Terraform driver or a Terraform Cloud driver. If there is a need to run both types of drivers, users will need to deploy a separate CTS instance for each type of driver. Relatedly, if there is a need to run CTS across multiple Terraform Cloud organizations, users will need to deploy a separate instance for each organization.
+Because a CTS instance can only be configured with one driver, an instance can only be associated with either a Terraform driver or a HCP Terraform driver. If there is a need to run both types of drivers, users will need to deploy a separate CTS instance for each type of driver. Relatedly, if there is a need to run CTS across multiple HCP Terraform organizations, users will need to deploy a separate instance for each organization.
 
 ### Required Setup
 
-This section captures requirements for setting up CTS to integrate with your [Terraform Cloud](https://cloud.hashicorp.com/products/terraform) solution.
+This section captures requirements for setting up CTS to integrate with your [HCP Terraform](https://cloud.hashicorp.com/products/terraform) solution.
 
-1. Hostname of your Terraform Cloud, self-hosted distribution
+1. Hostname of your HCP Terraform, self-hosted distribution
 1. Name of your organization
-1. [Team API token](/terraform/cloud-docs/users-teams-organizations/api-tokens) used for authentication with Terraform Cloud
+1. [Team API token](/terraform/cloud-docs/users-teams-organizations/api-tokens) used for authentication with HCP Terraform
 
-Prior to running CTS with a Terraform Cloud driver, you will need an account and organization set up, as well as a dedicated token. We recommend using a team token that is restricted to [Manage Workspaces](/terraform/cloud-docs/users-teams-organizations/teams#managing-workspace-access)-level permissions. Below are the steps for the recommended setup.
+Prior to running CTS with an HCP Terraform driver, you will need an account and organization set up, as well as a dedicated token. We recommend using a team token that is restricted to [Manage Workspaces](/terraform/cloud-docs/users-teams-organizations/teams#managing-workspace-access)-level permissions. Below are the steps for the recommended setup.
 
-The first step is to create an account with your Terraform Cloud service. After creating an account, create a new [organization](/terraform/cloud-docs/users-teams-organizations/organizations#creating-organizations) or select an existing organization. The address of your Terraform Cloud service will be used to configure the [`hostname`](/consul/docs/nia/configuration#hostname), and the organization name will be used to configure the [`organization`](/consul/docs/nia/configuration#organization) on the Terraform Cloud driver.
+The first step is to create an account with your HCP Terraform service. After creating an account, create a new [organization](/terraform/cloud-docs/users-teams-organizations/organizations#creating-organizations) or select an existing organization. The address of your HCP Terraform service will be used to configure the [`hostname`](/consul/docs/nia/configuration#hostname), and the organization name will be used to configure the [`organization`](/consul/docs/nia/configuration#organization) on the HCP Terraform driver.
 
 Once you have an account and organization, the next step is to [create a team](/terraform/cloud-docs/users-teams-organizations/teams). We recommend using a dedicated team and team token to run and authenticate CTS. Using a team token has the benefits of restricting organization permissions as well as associating CTS automated actions with the team rather than an individual.
 
@@ -127,12 +127,12 @@ After creating a dedicated team, update the team's permissions with "Manage Work
 
 [![CTS Terraform Team Setup](/img/nia/cts-tfc-team-setup.png)](/img/nia/cts-tfc-team-setup.png)
 
-After setting the team's permissions, the final setup step is to [generate the associated team token](/terraform/cloud-docs/users-teams-organizations/api-tokens), which can be done on the same team management page. This token will be used by CTS for API authentication and will be used to configure the [`token`](/consul/docs/nia/configuration#token) on the Terraform Cloud driver.
+After setting the team's permissions, the final setup step is to [generate the associated team token](/terraform/cloud-docs/users-teams-organizations/api-tokens), which can be done on the same team management page. This token will be used by CTS for API authentication and will be used to configure the [`token`](/consul/docs/nia/configuration#token) on the HCP Terraform driver.
 
 ### Recommendations
 
 We recommend configuring workspaces managed by CTS with [run notifications](/terraform/cloud-docs/workspaces/settings/notifications) through the Terraform web application. Run notifications notify external systems about the progress of runs and could help notify users of CTS events, particularly errored runs.
 
-[![CTS Terraform Cloud Run Notifications](/img/nia/cts-tfc-run-notifications.png)](/img/nia/cts-tfc-run-notifications.png)
+[![CTS HCP Terraform Run Notifications](/img/nia/cts-tfc-run-notifications.png)](/img/nia/cts-tfc-run-notifications.png)
 
 In order to configure a run notification, users can [manually create a notification configuration](/terraform/cloud-docs/workspaces/settings/notifications#creating-a-notification-configuration) for workspaces automated by CTS. A workspace may already exist for a task if the workspace name is identical to the configured task's [`name`](/consul/docs/nia/configuration#name-2). This may occur if CTS has already already run and created the workspace for the task. This may also occur if the workspace is manually created for the task prior to CTS running.

website/content/docs/nia/network-drivers/index.mdx

@@ -2,7 +2,7 @@
 layout: docs
 page_title: Network Drivers
 description: >-
-  Consul-Terraform-Sync Network Drivers with Terraform and Terraform Cloud
+  Consul-Terraform-Sync Network Drivers with Terraform and HCP Terraform
 ---
 
 # Network Drivers
@@ -11,18 +11,18 @@ Consul-Terraform-Sync (CTS) uses network drivers to execute and update network i
 
 CTS is a HashiCorp solution to Network Infrastructure Automation. It bridges Consul's networking features and Terraform infrastructure management capabilities. The solution seamlessly embeds Terraform as network drivers to manage automation of Terraform modules. This expands the Consul ecosystem and taps into the rich features and community of Terraform and Terraform providers.
 
-The following table highlights some of the additional features Terraform and Terraform Cloud offer when used as a network driver for CTS. Visit the [Terraform product page](https://www.hashicorp.com/products/terraform) or [contact our sales team](https://www.hashicorp.com/contact-sales) for a comprehensive list of features.
+The following table highlights some of the additional features Terraform and HCP Terraform offer when used as a network driver for CTS. Visit the [Terraform product page](https://www.hashicorp.com/products/terraform) or [contact our sales team](https://www.hashicorp.com/contact-sales) for a comprehensive list of features.
 
 | Network Driver | Description | Features |
 | -------------- | ----------- | -------- |
 | [Terraform driver](/consul/docs/nia/network-drivers/terraform) | CTS automates a local installation of the [Terraform CLI](https://www.terraform.io/) | - Local Terraform execution <br/> - Local workspace directories <br/> - [Backend options](/consul/docs/nia/configuration#backend) available for state storage <br/> |
-| [Terraform Cloud driver](/consul/docs/nia/network-drivers/terraform-cloud) | CTS Enterprise automates remote workspaces on [Terraform Cloud](/terraform/cloud-docs) | - [Remote Terraform execution](/terraform/cloud-docs/run/remote-operations) <br/> - Concurrent runs <br/> - [Secured variables](/terraform/cloud-docs/workspaces/variables) <br/> - [State versions](/terraform/cloud-docs/workspaces/state) <br/> - [Sentinel](/terraform/cloud-docs/policy-enforcement) to enforce governance policies as code <br/> - Audit [logs](/terraform/enterprise/admin/infrastructure/logging) and [trails](/terraform/cloud-docs/api-docs/audit-trails) <br/> - Run [history](/terraform/cloud-docs/run/manage), [triggers](/terraform/cloud-docs/workspaces/settings/run-triggers), and [notifications](/terraform/cloud-docs/workspaces/settings/notifications) <br/> - [Terraform Cloud Agents](/terraform/cloud-docs/agents) |
+| [HCP Terraform driver](/consul/docs/nia/network-drivers/terraform-cloud) | CTS Enterprise automates remote workspaces on [HCP Terraform](/terraform/cloud-docs) | - [Remote Terraform execution](/terraform/cloud-docs/run/remote-operations) <br/> - Concurrent runs <br/> - [Secured variables](/terraform/cloud-docs/workspaces/variables) <br/> - [State versions](/terraform/cloud-docs/workspaces/state) <br/> - [Sentinel](/terraform/cloud-docs/policy-enforcement) to enforce governance policies as code <br/> - Audit [logs](/terraform/enterprise/admin/infrastructure/logging) and [trails](/terraform/cloud-docs/api-docs/audit-trails) <br/> - Run [history](/terraform/cloud-docs/run/manage), [triggers](/terraform/cloud-docs/workspaces/settings/run-triggers), and [notifications](/terraform/cloud-docs/workspaces/settings/notifications) <br/> - [Terraform Cloud Agents](/terraform/cloud-docs/agents) |
 
 ## Understanding Terraform Automation
 
 CTS automates Terraform execution using a templated configuration to carry out infrastructure changes. The auto-generated configuration leverages input variables sourced from Consul and builds on top of reusable Terraform modules published and maintained by HashiCorp partners and the community. CTS can also run your custom built modules that suit your team's specific network automation needs.
 
-The network driver for CTS determines how the Terraform automation operates. Visit the driver pages to read more about the [Terraform driver](/consul/docs/nia/network-drivers/terraform) and the [Terraform Cloud driver](/consul/docs/nia/network-drivers/terraform-cloud).
+The network driver for CTS determines how the Terraform automation operates. Visit the driver pages to read more about the [Terraform driver](/consul/docs/nia/network-drivers/terraform) and the [HCP Terraform driver](/consul/docs/nia/network-drivers/terraform-cloud).
 
 ### Upgrading Terraform
 

website/content/docs/nia/usage/requirements.mdx

@@ -15,7 +15,7 @@ The following components are required to run Consul-Terraform-Sync (CTS):
 
 You can add support for your network infrastructure through Terraform providers so that you can apply Terraform modules to implement network integrations.
 
-The following guidance is for running CTS using the Terraform driver. The Terraform Cloud driver<EnterpriseAlert inline /> has [additional prerequisites](/consul/docs/nia/network-drivers/terraform-cloud#setting-up-terraform-cloud-driver).
+The following guidance is for running CTS using the Terraform driver. The HCP Terraform driver<EnterpriseAlert inline /> has [additional prerequisites](/consul/docs/nia/network-drivers/terraform-cloud#setting-up-terraform-cloud-driver).
 
 ## Run a Consul cluster
 

website/content/docs/nia/usage/run-ha.mdx

@@ -35,7 +35,7 @@ The following diagram shows the CTS cluster state after the leader stops. CTS In
 
 - The time it takes for a new leader to be elected is determined by the `high_availability.cluster.storage.session_ttl` configuration. The minimum failover time is equal to the `session_ttl` value. The maximum failover time is double the `session_ttl` value.
 - If failover occurs during task execution, a new leader is elected. The new leader will attempt to run all tasks once before continuing to monitor for changes.
-- If using the [Terraform Cloud (TFC) driver](/consul/docs/nia/network-drivers/terraform-cloud), the task finishes and CTS starts a new leader that attempts to queue a run for each task in TFC in once-mode.
+- If using the [HCP Terraform driver](/consul/docs/nia/network-drivers/terraform-cloud), the task finishes and CTS starts a new leader that attempts to queue a run for each task in HCP Terraform in once-mode.
 - If using [Terraform driver](/consul/docs/nia/network-drivers/terraform), the task may complete depending on the cause of the failover. The new leader starts and attempts to run each task in [once-mode](/consul/docs/nia/cli/start#modes). Depending on the module and provider, the task may require manual intervention to fix any inconsistencies between the infrastructure and Terraform state.
 - If failover occurs when no task is executing, CTS elects a new leader that attempts to run all tasks in once-mode.
 
@@ -51,7 +51,7 @@ Verify that you have met the [basic requirements](/consul/docs/nia/usage/require
 
 You must configure appropriate ACL permissions for your cluster. Refer to [ACL permissions](#) for details.
 
-We recommend specifying the [TFC driver](/consul/docs/nia/network-drivers/terraform-cloud) in your CTS configuration if you want to run in high availability mode.
+We recommend specifying the [HCP Terraform driver](/consul/docs/nia/network-drivers/terraform-cloud) in your CTS configuration if you want to run in high availability mode.
 
 ## Configuration
 

website/content/docs/release-notes/consul-terraform-sync/v0_6_x.mdx

@@ -12,7 +12,7 @@ description: >-
 
 We have implemented the following features in this release:
  
-### Per-task Execution Mode with Support for Terraform Cloud Agent <EnterpriseAlert inline /> 
+### Per-task Execution Mode with Support for HCP Terraform Agent <EnterpriseAlert inline /> 
 
 You can now execute Terraform tasks in `remote` or `cloud agent` mode. For more information, refer to the [per-task execution mode documentation](/consul/docs/nia/network-drivers/terraform-cloud#remote-workspaces).
 
@@ -38,7 +38,7 @@ CTS includes a new [API endpoint](/consul/docs/nia/api/health#health) that provi
 - Consul: v1.9+
 - <EnterpriseAlert inline /> HCP Consul: Latest
 - Terraform CLI: v0.13 - v1.1
-- <EnterpriseAlert inline /> Terraform Cloud: Latest
+- <EnterpriseAlert inline /> HCP Terraform: Latest
 - <EnterpriseAlert inline /> Terraform Enterprise: v202010-2 - Latest
 
 

website/content/docs/services/discovery/dns-cache.mdx

@@ -0,0 +1,239 @@
+---
+layout: docs
+page_title: Enable dynamic DNS queries
+description: ->
+    You tune Consul DNS query handling to balance between current information and reducing request response time. Learn how to enable caching by modifying TTL values, how to return stale results from the DNS cache, and how to configure Consul for negative response caching.
+---
+
+# DNS caching
+
+This page describes the process to return cached results in response to DNS lookups. Consul agents can use DNS caching to reduce response time, but might provide stale information in the process.
+
+## Introduction
+
+By default, Consul serves all DNS results with a `0` TTL value, which prevents any 
+caching. This configuration returns the most recent information because each DNS lookup 
+runs every time. However, this configuration adds latency to each lookup and can potentially 
+exhaust the query throughput of a datacenter.
+
+There are several ways you can modify to fine-tune Consul DNS lookup behavior to best suit your network's requirements.
+
+## TTL values ((#ttl))
+
+You can configure TTL values in the [agent configuration file](/consul/docs/agent/config/config-files) to allow DNS results to be cached downstream of Consul. 
+Higher TTL values reduce the number of lookups on the Consul servers and speed 
+lookups for clients, at the cost of increasingly stale results. By default, all 
+TTLs are zero, preventing any caching.
+
+<CodeTabs>
+
+```hcl
+dns_config {
+  service_ttl {
+    "*" = "0s"
+  }
+  node_ttl = "0s"
+}
+```
+
+```json
+{
+  "dns_config": {
+    "service_ttl": {
+      "*": "0s"
+    },
+    "node_ttl": "0s"
+  }
+}
+```
+
+</CodeTabs>
+
+### Enable caching
+
+To enable caching of node lookups, set the
+[`dns_config.node_ttl`](/consul/docs/agent/config/config-files#node_ttl) 
+value. This can be set to `10s` for example, and all node lookups will serve 
+results with a 10 second TTL.
+
+Service TTLs can be specified in a more granular fashion. You can set TTLs
+per-service, with a wildcard TTL as the default. This is specified using the
+[`dns_config.service_ttl`](/consul/docs/agent/config/config-files#service_ttl)
+map. The `*` is supported at the end of any prefix and has a lower precedence
+than strict match, so `my-service-x` has precedence over `my-service-*`. When
+performing wildcard match, the longest path is taken into account, thus
+`my-service-*` TTL will be used instead of `my-*` or `*`. With the same rule,
+`*` is the default value when nothing else matches. If no match is found the TTL
+defaults to 0.
+
+For example, a [`dns_config`](/consul/docs/agent/config/config-files#dns_config) 
+that provides a wildcard TTL and a specific TTL for a service might look like this:
+
+<CodeTabs>
+
+```hcl
+dns_config {
+  service_ttl {
+    "*" = "5s"
+    "web" = "30s"
+    "db*" = "10s"
+    "db-master" = "3s"
+  }
+}
+```
+
+```json
+{
+  "dns_config": {
+    "service_ttl": {
+      "*": "5s",
+      "web": "30s",
+      "db*": "10s",
+      "db-master": "3s"
+    }
+  }
+}
+```
+
+</CodeTabs>
+
+This sets all lookups to "web.service.consul" to use a 30 second TTL
+while lookups to "api.service.consul" will use the 5 second TTL from the wildcard.
+All lookups matching "db\*" would get a 10 seconds TTL except "db-master" that 
+would have a 3 seconds TTL.
+
+### Prepared queries
+
+[Prepared Queries](/consul/api-docs/query) provide an additional
+level of control over TTL. They allow for the TTL to be defined along with
+the query, and they can be changed on the fly by updating the query definition.
+If a TTL is not configured for a prepared query, then it will fall back to the
+service-specific configuration defined in the Consul agent as described above,
+and ultimately to 0 if no TTL is configured for the service in the Consul agent.
+
+<a name="stale"></a>
+
+## Stale reads
+
+Stale reads can be used to reduce latency and increase the throughput of DNS
+queries. The [settings](/consul/docs/agent/config/config-files) used to
+control stale reads of DNS queries are:
+
+- [`dns_config.allow_stale`](/consul/docs/agent/config/config-files#allow_stale) must be
+  set to true to enable stale reads.
+- [`dns_config.max_stale`](/consul/docs/agent/config/config-files#max_stale) limits how stale results
+  are allowed to be when querying DNS.
+
+With these two settings you can allow or prevent stale reads. Below we will 
+discuss the advantages and disadvantages of both.
+
+### Allow stale reads
+
+Since Consul 0.7.1, `allow_stale` is enabled by default and uses a `max_stale`
+value that defaults to a near-indefinite threshold (10 years). This allows DNS 
+queries to continue to be served in the event of a long outage with no leader. A 
+new telemetry counter has also been added at `consul.dns.stale_queries` to track 
+when agents serve DNS queries that are stale by more than 5 seconds.
+
+<CodeTabs>
+
+```hcl
+dns_config {
+  allow_stale = true
+  max_stale = "87600h"
+}
+```
+
+```json
+{
+  "dns_config": {
+    "allow_stale": true,
+    "max_stale": "87600h"
+  }
+}
+```
+
+</CodeTabs>
+
+
+<Note>
+
+ The above example is the default setting. You do not need to set it explicitly.
+
+</Note>
+
+Doing a stale read allows any Consul server to service a query, but non-leader 
+nodes may return data that is out-of-date. By allowing data to be slightly 
+stale, you get horizontal read scalability. Now any Consul server can service 
+the request, so you increase throughput by the number of servers in a datacenter.
+
+### Prevent stale reads
+
+If you want to prevent stale reads or limit how stale they can be, you can set 
+`allow_stale` to false or use a lower value for `max_stale`. Doing the first 
+will ensure that all reads are serviced by a 
+[single leader node](/consul/docs/architecture/consensus).
+The reads will then be strongly consistent but will be limited by the throughput
+of a single node.
+
+<CodeTabs>
+
+```hcl
+dns_config {
+  allow_stale = false
+}
+```
+
+```json
+{
+  "dns_config": {
+    "allow_stale": false
+  }
+}
+```
+
+</CodeTabs>
+
+## Negative response caching
+
+Although DNS clients cache negative responses, Consul returns a "not
+found" style response when a service exists but there are no healthy
+endpoints. When using DNS for service discovery, cached negative responses may
+cause a service to appear down for longer than it is actually unavailable.
+
+### Configure SOA
+
+In Consul v1.3.0 and newer, it is now possible to tune SOA responses and modify 
+the negative TTL cache for some resolvers. It can be achieved using the 
+[`soa.min_ttl`](/consul/docs/agent/config/config-files#soa_min_ttl)
+configuration within the [`soa`](/consul/docs/agent/config/config-files#soa) configuration.
+
+<CodeTabs>
+
+```hcl
+dns_config {
+  soa {
+    min_ttl = 60
+  }
+}
+```
+
+```json
+{
+  "dns_config": {
+    "soa": {
+      "min_ttl": 60
+    }
+  }
+}
+```
+
+</CodeTabs>
+
+One common example is that Windows will default to caching negative responses
+for 15 minutes. DNS forwarders may also cache negative responses, with the same
+effect. To avoid this problem, check the negative response cache defaults for
+your client operating system and any DNS forwarder on the path between the
+client and Consul and set the cache values appropriately. In many cases
+"appropriately" means turning negative response caching off to get the best
+recovery time when a service becomes available again.

website/content/docs/services/discovery/dns-configuration.mdx

@@ -65,12 +65,5 @@ machine.node.dc1.test-domain. 0 IN  TXT "consul-network-segment="
 Responses to pointer record (PTR) queries, such as `<ip>.in-addr.arpa.`, always use the [primary domain](/consul/docs/agent/config/config-files#domain) and not the alternative domain.
 
 ### Caching
-By default, DNS results served by Consul are not cached. Refer to the [DNS Caching tutorial](/consul/tutorials/networking/dns-caching) for instructions on how to enable caching.
-
-
-
-
-
-
-
 
+By default, DNS results served by Consul are not cached. Refer to [DNS caching](/consul/docs/services/discovery/dns-cache) for instructions on how to enable caching.

website/content/docs/upgrading/instructions/general-process.mdx

@@ -8,9 +8,7 @@ description: >-
 
 # General Upgrade Process
 
-## Introduction
-
-This document describes some best practices that you should follow when
+This pages describes the overall process and best practices that you should follow when
 upgrading Consul. Some versions also have steps that are specific to that
 version, so make sure you also review the [upgrade instructions](/consul/docs/upgrading/instructions)
 for the version you are on.
@@ -46,117 +44,124 @@ If you are using Kubernetes, then please review our documentation for
 
 ## Prepare for the Upgrade
 
-**1.** Take a snapshot:
+1. Take a snapshot to ensure you have a safe fallback option in case something goes wrong.
 
-```
-consul snapshot save backup.snap
-```
+  ```shell-session
+  $ consul snapshot save backup.snap
+  ```
 
-You can inspect the snapshot to ensure if was successful with:
+  You can inspect the snapshot to ensure your cluster's Raft index was successfully captured.
 
-```
-consul snapshot inspect backup.snap
-```
+  ```shell-session
+  $ consul snapshot inspect backup.snap
+  ```
 
-Example output:
+  Example output:
 
-```
-ID           2-1182-1542056499724
-Size         4115
-Index        1182
-Term         2
-Version      1
-```
+  ```shell-session hideClipboard
+  ID           2-1182-1542056499724
+  Size         4115
+  Index        1182
+  Term         2
+  Version      1
+  ```
 
-This will ensure you have a safe fallback option in case something goes wrong. Store
-this snapshot somewhere safe. More documentation on snapshot usage is available here:
+  Store this snapshot somewhere safe. For more information on snapshots, refer to the following:
 
-- [consul.io/commands/snapshot](/consul/commands/snapshot)
-- [Backup Consul Data and State tutorial](/consul/tutorials/production-deploy/backup-and-restore)
+  - [`consul snapshot` CLI command](/consul/commands/snapshot)
+  - [Backup Consul Data and State tutorial](/consul/tutorials/production-deploy/backup-and-restore)
 
-**2.** Temporarily modify your Consul configuration so that its [log_level](/consul/docs/agent/config/cli-flags#_log_level)
-is set to `debug`. After doing this, issue the following command on your servers to
+2. Temporarily modify your Consul configuration to set the agent's [`log_level`](/consul/docs/agent/config/cli-flags#_log_level) is set to `debug`. Then issue the following command on your servers to
 reload the configuration:
 
-```
-consul reload
+```shell-session
+$ consul reload
 ```
 
-This change will give you more information to work with in the event something goes wrong.
+When you change the cluster's log level, Consul gives you more information to work with in the event that something goes wrong.
+
+### Enterprise upgrades
+
+<Note>
+To experience a smoother upgrade process on Consul Enterprise, we recommend that you disable the upgrade migration feature.
+</Note>
+
+Consul Enterprise supports [automated upgrades](/consul/docs/enterprise/upgrades), but the autopilot feature may cause a node running an updated Consul version to elect a new leader before the version is updated on the existing cluster leader.
+
+If your datacenter runs Consul Enterprise, update your server agent configuration file to disable autopilot's upgrade migration or run the following CLI command:
+
+```shell-session
+$ consul operator autopilot set-config -disable-upgrade-migration=true
+```
 
 ## Perform the Upgrade
 
-**1.** Issue the following command to discover which server is currently the leader:
+1. Issue the following command to discover which server is currently the leader:
 
-```
-consul operator raft list-peers
-```
+  ```shell-session
+  $ consul operator raft list-peers
+  ```
 
-You should receive output similar to this (exact formatting and content may differ based on version):
+  You should receive output similar to this (exact formatting and content may differ based on version):
 
-```
-Node       ID                                    Address         State     Voter  RaftProtocol
-dc1-node1  ae15858f-7f5f-4dcb-b7d5-710fdcdd2745  10.11.0.2:8300  leader    true   3
-dc1-node2  20e6be1b-f1cb-4aab-929f-f7d2d43d9a96  10.11.0.3:8300  follower  true   3
-dc1-node3  658c343b-8769-431f-a71a-236f9dbb17b3  10.11.0.4:8300  follower  true   3
-```
+  ```shell-session hideClipboard
+  Node       ID                                    Address         State     Voter  RaftProtocol
+  dc1-node1  ae15858f-7f5f-4dcb-b7d5-710fdcdd2745  10.11.0.2:8300  leader    true   3
+  dc1-node2  20e6be1b-f1cb-4aab-929f-f7d2d43d9a96  10.11.0.3:8300  follower  true   3
+  dc1-node3  658c343b-8769-431f-a71a-236f9dbb17b3  10.11.0.4:8300  follower  true   3
+  ```
 
-Take note of which agent is the leader.
+  Take note of which agent is the leader.
 
-**2.** Copy the new `consul` binary onto your servers and replace the existing
-binary with the new one.
+2. Copy the new `consul` binary onto your servers and replace the existing binary with the new one.
 
-**3.** The following steps must be done in order on the server agents, leaving the leader
-agent for last. First, use a service management system (e.g., systemd, upstart, etc.) to restart the Consul service. If
-you are not using a service management system, you must restart the agent manually.
+3. Use a service management system such as systemd or upstart to restart the Consul service on each server. You must restart follower server agents first, leaving the leader agent for last. If you are not using a service management system, you must restart the agent manually.
 
-To validate that the agent has rejoined the cluster and is in sync with the leader, issue the
-following command:
+  To validate that the agent has rejoined the cluster and is in sync with the leader after you restart it, issue the following command to the agent:
 
-```
-consul info
-```
+  ```shell-session
+  $ consul info
+  ```
 
-Check whether the `commit_index` and `last_log_index` fields have the same value. If done properly,
-this should avoid an unexpected leadership election due to loss of quorum.
+  Check whether the `commit_index` and `last_log_index` fields have the same value. If done properly,
+  this should avoid an unexpected leadership election due to loss of quorum.
 
-**4.** Double-check that all servers are showing up in the cluster as expected and are on
-the correct version by issuing:
+4. Double-check that all servers joined the cluster as expected and run the correct version. Issue the following command:
 
-```
-consul members
-```
+  ```shell-session
+  $ consul members
+  ```
 
-You should receive output similar to this:
+  You should receive output that lists the servers as `alive` and running the same updated Consul version.
 
-```
-Node       Address         Status  Type    Build  Protocol  DC
-dc1-node1  10.11.0.2:8301  alive   server  1.8.3  2         dc1
-dc1-node2  10.11.0.3:8301  alive   server  1.8.3  2         dc1
-dc1-node3  10.11.0.4:8301  alive   server  1.8.3  2         dc1
-```
+  ```shell-session hideClipboard
+  Node       Address         Status  Type    Build  Protocol  DC
+  dc1-node1  10.11.0.2:8301  alive   server  1.8.3  2         dc1
+  dc1-node2  10.11.0.3:8301  alive   server  1.8.3  2         dc1
+  dc1-node3  10.11.0.4:8301  alive   server  1.8.3  2         dc1
+  ```
 
-Also double-check the raft state to make sure there is a leader and sufficient voters:
+  Also double-check the Raft state to make sure there is a leader and sufficient voters:
 
-```
-consul operator raft list-peers
-```
+  ```shell-session
+  $ consul operator raft list-peers
+  ```
 
-You should receive output similar to this:
+  You should receive output that lists one server as the `leader` and the rest as `follower`:
 
-```
-Node       ID                                    Address         State     Voter  RaftProtocol
-dc1-node1  ae15858f-7f5f-4dcb-b7d5-710fdcdd2745  10.11.0.2:8300  leader    true   3
-dc1-node2  20e6be1b-f1cb-4aab-929f-f7d2d43d9a96  10.11.0.3:8300  follower  true   3
-dc1-node3  658c343b-8769-431f-a71a-236f9dbb17b3  10.11.0.4:8300  follower  true   3
-```
+  ```shell-session hideClipboard
+  Node       ID                                    Address         State     Voter  RaftProtocol
+  dc1-node1  ae15858f-7f5f-4dcb-b7d5-710fdcdd2745  10.11.0.2:8300  leader    true   3
+  dc1-node2  20e6be1b-f1cb-4aab-929f-f7d2d43d9a96  10.11.0.3:8300  follower  true   3
+  dc1-node3  658c343b-8769-431f-a71a-236f9dbb17b3  10.11.0.4:8300  follower  true   3
+  ```
 
-**5.** Set your `log_level` back to its original value and issue the following command
+5. Set your `log_level` back to its original value and issue the following command
 on your servers to reload the configuration:
 
-```
-consul reload
-```
+  ```shell-session
+  $ consul reload
+  ```
 
 ## Troubleshooting
 
@@ -164,7 +169,7 @@ Most problems with upgrading occur due to either failing to upgrade the leader a
 or failing to wait for a follower agent to fully rejoin a cluster before moving
 on to another server. This can cause a loss of quorum and occasionally can result in
 all of your servers attempting to kick off leadership elections endlessly without ever
-reaching a quorum and electing a leader.
+reaching a quorum and electing a leader. Consul Enterprise users should [disable upgrade migration](#enterprise-upgrades) to prevent autopilot from prematurely electing a new cluster leader.
 
 Most of these problems can be solved by following the steps outlined in our
 [Disaster recovery for Consul clusters](/consul/tutorials/datacenter-operations/recovery-outage) document.

website/data/docs-nav-data.json

@@ -403,6 +403,10 @@
             "path": "services/discovery/dns-static-lookups"
           },
           {
+            "title": "Cache DNS lookups",
+            "path": "services/discovery/dns-cache"
+          },
+          {           
             "title": "Enable dynamic DNS lookups",
             "path": "services/discovery/dns-dynamic-lookups"
           }
@@ -465,6 +469,10 @@
             "title": "TCP route",
             "path": "connect/config-entries/tcp-route"
           },
+          {
+            "title": "File system certificate",
+            "path": "connect/config-entries/file-system-certificate"
+          },
           {
             "title": "Inline certificate",
             "path": "connect/config-entries/inline-certificate"
@@ -1021,7 +1029,16 @@
     "routes": [
       {
         "title": "Consul KV",
-        "path": "dynamic-app-config/kv"
+        "routes": [
+          {
+            "title": "Overview",
+            "path": "dynamic-app-config/kv"
+          },
+          {
+            "title": "Store keys and values",
+            "path": "dynamic-app-config/kv/store"
+          }
+        ]
       },
       {
         "title": "Sessions",
@@ -2043,7 +2060,7 @@
             "path": "nia/enterprise/license"
           },
           {
-            "title": "Terraform Cloud Driver",
+            "title": "HCP Terraform Driver",
             "href": "/docs/nia/network-drivers/terraform-cloud"
           }
         ]
@@ -2060,8 +2077,8 @@
             "path": "nia/network-drivers/terraform"
           },
           {
-            "title": "Terraform Cloud",
-            "path": "nia/network-drivers/terraform-cloud"
+            "title": "HCP Terraform",
+            "path": "nia/network-drivers/hcp-terraform"
           }
         ]
       },

website/redirects.js

@@ -242,4 +242,14 @@ module.exports = [
     destination: '/consul/docs/:version/architecture/catalog/v2/:slug',
     permanent: true
   },
+  {
+    source: '/consul/docs/nia/network-drivers/terraform-cloud',
+    destination: '/consul/docs/nia/network-drivers/hcp-terraform',
+    permanent: true
+  },
+  {
+    source: '/consul/docs/:version(v1\.(?:8|9|10|11|12|13|14|15|16|17)\.x)/nia/network-drivers/hcp-terraform',
+    destination: '/consul/docs/:version/nia/network-drivers/terraform-cloud',
+    permanent: true
+  },
 ]