agent: add primary_datacenter and connect replication config options

agent/agent.go

@@ -874,6 +874,7 @@ func (a *Agent) consulConfig() (*consul.Config, error) {
 	// todo(fs): these are now always set in the runtime config so we can simplify this
 	// todo(fs): or is there a reason to keep it like that?
 	base.Datacenter = a.config.Datacenter
+	base.PrimaryDatacenter = a.config.PrimaryDatacenter
 	base.DataDir = a.config.DataDir
 	base.NodeName = a.config.NodeName
 

agent/agent_endpoint.go

@@ -1152,6 +1152,9 @@ func (s *HTTPServer) AgentToken(resp http.ResponseWriter, req *http.Request) (in
 	case "acl_replication_token":
 		s.agent.tokens.UpdateACLReplicationToken(args.Token)
 
+	case "connect_replication_token":
+		s.agent.tokens.UpdateConnectReplicationToken(args.Token)
+
 	default:
 		resp.WriteHeader(http.StatusNotFound)
 		fmt.Fprintf(resp, "Token %q is unknown", target)

agent/config/builder.go

@@ -558,6 +558,15 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		})
 	}
 
+	primaryDatacenter := strings.ToLower(b.stringVal(c.PrimaryDatacenter))
+	if c.ACLDatacenter != nil {
+		b.warn("The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead.")
+
+		if primaryDatacenter == "" {
+			primaryDatacenter = strings.ToLower(b.stringVal(c.ACLDatacenter))
+		}
+	}
+
 	proxyDefaultExecMode := b.stringVal(c.Connect.ProxyDefaults.ExecMode)
 	proxyDefaultDaemonCommand := c.Connect.ProxyDefaults.DaemonCommand
 	proxyDefaultScriptCommand := c.Connect.ProxyDefaults.ScriptCommand
@@ -737,6 +746,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		NodeName:                                b.nodeName(c.NodeName),
 		NonVotingServer:                         b.boolVal(c.NonVotingServer),
 		PidFile:                                 b.stringVal(c.PidFile),
+		PrimaryDatacenter:                       primaryDatacenter,
 		RPCAdvertiseAddr:                        rpcAdvertiseAddr,
 		RPCBindAddr:                             rpcBindAddr,
 		RPCHoldTimeout:                          b.durationVal("performance.rpc_hold_timeout", c.Performance.RPCHoldTimeout),

agent/config/config.go

@@ -212,6 +212,7 @@ type Config struct {
 	Performance                      Performance              `json:"performance,omitempty" hcl:"performance" mapstructure:"performance"`
 	PidFile                          *string                  `json:"pid_file,omitempty" hcl:"pid_file" mapstructure:"pid_file"`
 	Ports                            Ports                    `json:"ports,omitempty" hcl:"ports" mapstructure:"ports"`
+	PrimaryDatacenter                *string                  `json:"primary_datacenter,omitempty" hcl:"primary_datacenter" mapstructure:"primary_datacenter"`
 	RPCProtocol                      *int                     `json:"protocol,omitempty" hcl:"protocol" mapstructure:"protocol"`
 	RaftProtocol                     *int                     `json:"raft_protocol,omitempty" hcl:"raft_protocol" mapstructure:"raft_protocol"`
 	RaftSnapshotThreshold            *int                     `json:"raft_snapshot_threshold,omitempty" hcl:"raft_snapshot_threshold" mapstructure:"raft_snapshot_threshold"`
@@ -484,11 +485,12 @@ type Upstream struct {
 type Connect struct {
 	// Enabled opts the agent into connect. It should be set on all clients and
 	// servers in a cluster for correct connect operation.
-	Enabled       *bool                  `json:"enabled,omitempty" hcl:"enabled" mapstructure:"enabled"`
+	Enabled          *bool                  `json:"enabled,omitempty" hcl:"enabled" mapstructure:"enabled"`
-	Proxy         ConnectProxy           `json:"proxy,omitempty" hcl:"proxy" mapstructure:"proxy"`
+	Proxy            ConnectProxy           `json:"proxy,omitempty" hcl:"proxy" mapstructure:"proxy"`
-	ProxyDefaults ConnectProxyDefaults   `json:"proxy_defaults,omitempty" hcl:"proxy_defaults" mapstructure:"proxy_defaults"`
+	ProxyDefaults    ConnectProxyDefaults   `json:"proxy_defaults,omitempty" hcl:"proxy_defaults" mapstructure:"proxy_defaults"`
-	CAProvider    *string                `json:"ca_provider,omitempty" hcl:"ca_provider" mapstructure:"ca_provider"`
+	CAProvider       *string                `json:"ca_provider,omitempty" hcl:"ca_provider" mapstructure:"ca_provider"`
-	CAConfig      map[string]interface{} `json:"ca_config,omitempty" hcl:"ca_config" mapstructure:"ca_config"`
+	CAConfig         map[string]interface{} `json:"ca_config,omitempty" hcl:"ca_config" mapstructure:"ca_config"`
+	ReplicationToken *string                `json:"replication_token,omitempty" hcl:"replication_token" mapstructure:"replication_token"`
 }
 
 // ConnectProxy is the agent-global connect proxy configuration.

agent/config/runtime.go

@@ -503,6 +503,9 @@ type RuntimeConfig struct {
 	// ConnectCAConfig is the config to use for the CA provider.
 	ConnectCAConfig map[string]interface{}
 
+	// ConnectReplicationToken is the ACL token used for replicating intentions.
+	ConnectReplicationToken string
+
 	// ConnectTestDisableManagedProxies is not exposed to public config but us
 	// used by TestAgent to prevent self-executing the test binary in the
 	// background if a managed proxy is created for a test. The only place we
@@ -800,6 +803,13 @@ type RuntimeConfig struct {
 	// hcl: pid_file = string
 	PidFile string
 
+	// PrimaryDatacenter is the central datacenter that holds authoritative
+	// ACL records, replicates intentions and holds the root CA for Connect.
+	// This must be the same for the entire cluster. Off by default.
+	//
+	// hcl: primary_datacenter = string
+	PrimaryDatacenter string
+
 	// RPCAdvertiseAddr is the TCP address Consul advertises for its RPC endpoint.
 	// By default this is the bind address on the default RPC Server port. If the
 	// advertise address is specified then it is used.

agent/config/runtime_test.go

@@ -1379,7 +1379,9 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 			patch: func(rt *RuntimeConfig) {
 				rt.ACLDatacenter = "a"
 				rt.DataDir = dataDir
+				rt.PrimaryDatacenter = "a"
 			},
+			warns: []string{`The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead.`},
 		},
 		{
 			desc: "acl_replication_token enables acl replication",
@@ -1472,9 +1474,10 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				`-datacenter=a`,
 				`-data-dir=` + dataDir,
 			},
-			json: []string{`{ "acl_datacenter": "%" }`},
+			json:  []string{`{ "acl_datacenter": "%" }`},
-			hcl:  []string{`acl_datacenter = "%"`},
+			hcl:   []string{`acl_datacenter = "%"`},
-			err:  `acl_datacenter cannot be "%". Please use only [a-z0-9-_]`,
+			err:   `acl_datacenter cannot be "%". Please use only [a-z0-9-_]`,
+			warns: []string{`The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead.`},
 		},
 		{
 			desc: "autopilot.max_trailing_logs invalid",
@@ -3013,6 +3016,7 @@ func TestFullConfig(t *testing.T) {
 				"sidecar_max_port": 9999
 			},
 			"protocol": 30793,
+			"primary_datacenter": "ejtmd43d",
 			"raft_protocol": 19016,
 			"raft_snapshot_threshold": 16384,
 			"raft_snapshot_interval": "30s",
@@ -3543,6 +3547,7 @@ func TestFullConfig(t *testing.T) {
 				sidecar_max_port = 9999
 			}
 			protocol = 30793
+			primary_datacenter = "ejtmd43d"
 			raft_protocol = 19016
 			raft_snapshot_threshold = 16384
 			raft_snapshot_interval = "30s"
@@ -4146,6 +4151,7 @@ func TestFullConfig(t *testing.T) {
 		NodeName:                         "otlLxGaI",
 		NonVotingServer:                  true,
 		PidFile:                          "43xN80Km",
+		PrimaryDatacenter:                "ejtmd43d",
 		RPCAdvertiseAddr:                 tcpAddr("17.99.29.16:3757"),
 		RPCBindAddr:                      tcpAddr("16.99.34.17:3757"),
 		RPCHoldTimeout:                   15707 * time.Second,
@@ -4488,6 +4494,7 @@ func TestFullConfig(t *testing.T) {
 	}
 
 	warns := []string{
+		`The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead.`,
 		`bootstrap_expect > 0: expecting 53 servers`,
 	}
 
@@ -4849,6 +4856,7 @@ func TestSanitize(t *testing.T) {
 		"ConnectProxyDefaultScriptCommand": [],
 		"ConnectSidecarMaxPort": 0,
 		"ConnectSidecarMinPort": 0,
+		"ConnectReplicationToken": "hidden",
 		"ConnectTestDisableManagedProxies": false,
 		"ConsulCoordinateUpdateBatchSize": 0,
 		"ConsulCoordinateUpdateMaxBatches": 0,
@@ -4931,6 +4939,7 @@ func TestSanitize(t *testing.T) {
 		"NodeName": "",
 		"NonVotingServer": false,
 		"PidFile": "",
+		"PrimaryDatacenter": "",
 		"RPCAdvertiseAddr": "",
 		"RPCBindAddr": "",
 		"RPCHoldTimeout": "0s",

agent/consul/config.go

@@ -75,10 +75,14 @@ type Config struct {
 	// of nodes.
 	BootstrapExpect int
 
-	// Datacenter is the datacenter this Consul server represents
+	// Datacenter is the datacenter this Consul server represents.
 	Datacenter string
 
-	// DataDir is the directory to store our state in
+	// PrimaryDatacenter is the authoritative datacenter for features like ACLs
+	// and Connect.
+	PrimaryDatacenter string
+
+	// DataDir is the directory to store our state in.
 	DataDir string
 
 	// DevMode is used to enable a development server mode.

agent/token/store.go

@@ -30,6 +30,10 @@ type Store struct {
 	// aclReplicationToken is a special token that's used by servers to
 	// replicate ACLs from the ACL datacenter.
 	aclReplicationToken string
+
+	// connectReplicationToken is a special token that's used by servers to
+	// replicate intentions from the primary datacenter.
+	connectReplicationToken string
 }
 
 // UpdateUserToken replaces the current user token in the store.
@@ -60,6 +64,13 @@ func (t *Store) UpdateACLReplicationToken(token string) {
 	t.l.Unlock()
 }
 
+// UpdateConnectReplicationToken replaces the current Connect replication token in the store.
+func (t *Store) UpdateConnectReplicationToken(token string) {
+	t.l.Lock()
+	t.connectReplicationToken = token
+	t.l.Unlock()
+}
+
 // UserToken returns the best token to use for user operations.
 func (t *Store) UserToken() string {
 	t.l.RLock()
@@ -87,6 +98,14 @@ func (t *Store) ACLReplicationToken() string {
 	return t.aclReplicationToken
 }
 
+// ConnectReplicationToken returns the Connect replication token.
+func (t *Store) ConnectReplicationToken() string {
+	t.l.RLock()
+	defer t.l.RUnlock()
+
+	return t.connectReplicationToken
+}
+
 // IsAgentMasterToken checks to see if a given token is the agent master token.
 // This will never match an empty token for safety.
 func (t *Store) IsAgentMasterToken(token string) bool {

website/source/docs/guides/acl.html.md

@@ -1137,7 +1137,7 @@ Consul clients must have [`primary_datacenter`](/docs/agent/options.html#primary
 in order to enable agent-level ACL features. If this is set, the agents will contact the Consul
 servers to determine if ACLs are enabled at the cluster level. If they detect that ACLs are not
 enabled, they will check at most every 2 minutes to see if they have become enabled, and will
-start enforcing ACLs automatically. If an agent has an `acl_daprimary_datacentertacenter` defined, operators will
+start enforcing ACLs automatically. If an agent has an `primary_datacenter` defined, operators will
 need to use the [`acl_agent_master_token`](/docs/agent/options.html#acl_agent_master_token) to
 perform agent-level operations if the Consul servers aren't present (such as for a manual join
 to the cluster), unless the [`acl_down_policy`](/docs/agent/options.html#acl_down_policy) on the