Refactor AutoConfig RPC to not have a direct dependency on the Server type

Instead it has an interface which can be mocked for better unit testing that is deterministic and not prone to flakiness.

agent/agentpb/auto_config.pb.go

@@ -23,7 +23,7 @@ var _ = math.Inf
 const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 // AutoConfigRequest is the data structure to be sent along with the
-// Cluster.AutoConfig RPC
+// AutoConfig.InitialConfiguration RPC
 type AutoConfigRequest struct {
 	// Datacenter is the local datacenter name. This wont actually be set by clients
 	// but rather will be set by the servers to allow for forwarding to
@@ -113,7 +113,7 @@ func (m *AutoConfigRequest) GetConsulToken() string {
 	return ""
 }
 
-// AutoConfigResponse is the data structure sent in response to a Cluster.AutoConfig request
+// AutoConfigResponse is the data structure sent in response to a AutoConfig.InitialConfiguration request
 type AutoConfigResponse struct {
 	Config               *config.Config `protobuf:"bytes,1,opt,name=Config,proto3" json:"Config,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}       `json:"-"`

agent/agentpb/auto_config.proto

@@ -7,7 +7,7 @@ option go_package = "github.com/hashicorp/consul/agent/agentpb";
 import "agent/agentpb/config/config.proto";
 
 // AutoConfigRequest is the data structure to be sent along with the 
-// Cluster.AutoConfig RPC
+// AutoConfig.InitialConfiguration RPC
 message AutoConfigRequest {
    // Datacenter is the local datacenter name. This wont actually be set by clients
    // but rather will be set by the servers to allow for forwarding to
@@ -30,7 +30,7 @@ message AutoConfigRequest {
    string ConsulToken = 6;
 }
 
-// AutoConfigResponse is the data structure sent in response to a Cluster.AutoConfig request
+// AutoConfigResponse is the data structure sent in response to a AutoConfig.InitialConfiguration request
 message AutoConfigResponse {
    config.Config Config = 1;
 }

agent/auto-config/auto_config.go

@@ -279,7 +279,7 @@ func (ac *AutoConfig) InitialConfiguration(ctx context.Context) (*config.Runtime
 }
 
 // introToken is responsible for determining the correct intro token to use
-// when making the initial Cluster.AutoConfig RPC request.
+// when making the initial AutoConfig.InitialConfiguration RPC request.
 func (ac *AutoConfig) introToken() (string, error) {
 	conf := ac.config.AutoConfig
 	// without an intro token or intro token file we cannot do anything
@@ -431,7 +431,7 @@ func (ac *AutoConfig) recordAutoConfigReply(reply *agentpb.AutoConfigResponse) e
 }
 
 // getInitialConfigurationOnce will perform full server to TCPAddr resolution and
-// loop through each host trying to make the Cluster.AutoConfig RPC call. When
+// loop through each host trying to make the AutoConfig.InitialConfiguration RPC call. When
 // successful the bool return will be true and the err value will indicate whether we
 // successfully recorded the auto config settings (persisted to disk and stored internally
 // on the AutoConfig object)
@@ -462,9 +462,9 @@ func (ac *AutoConfig) getInitialConfigurationOnce(ctx context.Context) (bool, er
 				return false, ctx.Err()
 			}
 
-			ac.logger.Debug("making Cluster.AutoConfig RPC", "addr", addr.String())
+			ac.logger.Debug("making AutoConfig.InitialConfiguration RPC", "addr", addr.String())
-			if err = ac.directRPC.RPC(ac.config.Datacenter, ac.config.NodeName, &addr, "Cluster.AutoConfig", &request, &reply); err != nil {
+			if err = ac.directRPC.RPC(ac.config.Datacenter, ac.config.NodeName, &addr, "AutoConfig.InitialConfiguration", &request, &reply); err != nil {
-				ac.logger.Error("AutoConfig RPC failed", "addr", addr.String(), "error", err)
+				ac.logger.Error("AutoConfig.InitialConfiguration RPC failed", "addr", addr.String(), "error", err)
 				continue
 			}
 

agent/auto-config/auto_config_test.go

@@ -207,7 +207,7 @@ func TestInitialConfiguration_cancelled(t *testing.T) {
 		JWT:        "blarg",
 	}
 
-	directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "Cluster.AutoConfig", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
+	directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "AutoConfig.InitialConfiguration", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
 	ac, err := New(WithBuilderOpts(builderOpts), WithTLSConfigurator(&tlsutil.Configurator{}), WithDirectRPC(&directRPC))
 	require.NoError(t, err)
 	require.NotNil(t, ac)
@@ -289,7 +289,7 @@ func TestInitialConfiguration_success(t *testing.T) {
 		"dc1",
 		"autoconf",
 		&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300},
-		"Cluster.AutoConfig",
+		"AutoConfig.InitialConfiguration",
 		&expectedRequest,
 		&agentpb.AutoConfigResponse{}).Return(populateResponse)
 
@@ -344,7 +344,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
 		"dc1",
 		"autoconf",
 		&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
-		"Cluster.AutoConfig",
+		"AutoConfig.InitialConfiguration",
 		&expectedRequest,
 		&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
 	directRPC.On(
@@ -352,7 +352,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
 		"dc1",
 		"autoconf",
 		&net.TCPAddr{IP: net.IPv4(198, 18, 0, 2), Port: 8398},
-		"Cluster.AutoConfig",
+		"AutoConfig.InitialConfiguration",
 		&expectedRequest,
 		&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
 	directRPC.On(
@@ -360,7 +360,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
 		"dc1",
 		"autoconf",
 		&net.TCPAddr{IP: net.IPv4(198, 18, 0, 3), Port: 8399},
-		"Cluster.AutoConfig",
+		"AutoConfig.InitialConfiguration",
 		&expectedRequest,
 		&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
 	directRPC.On(
@@ -368,7 +368,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
 		"dc1",
 		"autoconf",
 		&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
-		"Cluster.AutoConfig",
+		"AutoConfig.InitialConfiguration",
 		&expectedRequest,
 		&agentpb.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Once()
 	directRPC.On(
@@ -376,7 +376,7 @@ func TestInitialConfiguration_retries(t *testing.T) {
 		"dc1",
 		"autoconf",
 		&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
-		"Cluster.AutoConfig",
+		"AutoConfig.InitialConfiguration",
 		&expectedRequest,
 		&agentpb.AutoConfigResponse{}).Return(populateResponse)
 

agent/consul/auto_config_endpoint.go

@@ -4,16 +4,12 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
-	"net"
-	"time"
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/agentpb"
 	"github.com/hashicorp/consul/agent/agentpb/config"
 	"github.com/hashicorp/consul/agent/consul/authmethod/ssoauth"
-	"github.com/hashicorp/consul/agent/metadata"
 	"github.com/hashicorp/consul/agent/structs"
-	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/lib/template"
 	"github.com/hashicorp/consul/tlsutil"
 	bexpr "github.com/hashicorp/go-bexpr"
@@ -55,7 +51,6 @@ func (a *jwtAuthorizer) Authorize(req *agentpb.AutoConfigRequest) (AutoConfigOpt
 		"segment": req.Segment,
 	}
 
-	// TODO (autoconf) check for JWT reuse if configured to do so.
 	for _, raw := range a.claimAssertions {
 		// validate and fill any HIL
 		filled, err := template.InterpolateHIL(raw, varMap, true)
@@ -84,87 +79,86 @@ func (a *jwtAuthorizer) Authorize(req *agentpb.AutoConfigRequest) (AutoConfigOpt
 	}, nil
 }
 
-// Cluster endpoint is used for cluster configuration operations
+type AutoConfigBackend interface {
-type Cluster struct {
+	GetConfig() *Config
-	srv *Server
+	CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error)
+	DatacenterJoinAddresses(segment string) ([]string, error)
+	TLSConfigurator() *tlsutil.Configurator
+	ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error)
+}
 
+// AutoConfig endpoint is used for cluster auto configuration operations
+type AutoConfig struct {
+	backend    AutoConfigBackend
 	authorizer AutoConfigAuthorizer
 }
 
+// getBackendConfig is a small wrapper around the backend's GetConfig to ensure
+// that all helper functions have a non-nil configuration to operate on.
+func (ac *AutoConfig) getBackendConfig() *Config {
+	if conf := ac.backend.GetConfig(); conf != nil {
+		return conf
+	}
+
+	return DefaultConfig()
+}
+
 // updateTLSCertificatesInConfig will ensure that the TLS settings regarding how an agent is
 // made aware of its certificates are populated. This will only work if connect is enabled and
 // in some cases only if auto_encrypt is enabled on the servers. This endpoint has the option
 // to configure auto_encrypt or potentially in the future to generate the certificates inline.
-func (c *Cluster) updateTLSCertificatesInConfig(opts AutoConfigOptions, conf *config.Config) error {
+func (ac *AutoConfig) updateTLSCertificatesInConfig(opts AutoConfigOptions, conf *config.Config) error {
-	if c.srv.config.AutoEncryptAllowTLS {
+	conf.AutoEncrypt = &config.AutoEncrypt{TLS: ac.getBackendConfig().AutoEncryptAllowTLS}
-		conf.AutoEncrypt = &config.AutoEncrypt{TLS: true}
-	} else {
-		conf.AutoEncrypt = &config.AutoEncrypt{TLS: false}
-	}
-
 	return nil
 }
 
 // updateACLtokensInConfig will configure all of the agents ACL settings and will populate
 // the configuration with an agent token usable for all default agent operations.
-func (c *Cluster) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config) error {
+func (ac *AutoConfig) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config) error {
+	backendConf := ac.getBackendConfig()
+
 	acl := &config.ACL{
-		Enabled:             c.srv.config.ACLsEnabled,
+		Enabled:             backendConf.ACLsEnabled,
-		PolicyTTL:           c.srv.config.ACLPolicyTTL.String(),
+		PolicyTTL:           backendConf.ACLPolicyTTL.String(),
-		RoleTTL:             c.srv.config.ACLRoleTTL.String(),
+		RoleTTL:             backendConf.ACLRoleTTL.String(),
-		TokenTTL:            c.srv.config.ACLTokenTTL.String(),
+		TokenTTL:            backendConf.ACLTokenTTL.String(),
-		DisabledTTL:         c.srv.config.ACLDisabledTTL.String(),
+		DisabledTTL:         backendConf.ACLDisabledTTL.String(),
-		DownPolicy:          c.srv.config.ACLDownPolicy,
+		DownPolicy:          backendConf.ACLDownPolicy,
-		DefaultPolicy:       c.srv.config.ACLDefaultPolicy,
+		DefaultPolicy:       backendConf.ACLDefaultPolicy,
-		EnableKeyListPolicy: c.srv.config.ACLEnableKeyListPolicy,
+		EnableKeyListPolicy: backendConf.ACLEnableKeyListPolicy,
 	}
 
 	// when ACLs are enabled we want to create a local token with a node identity
-	if c.srv.config.ACLsEnabled {
+	if backendConf.ACLsEnabled {
-		// we have to require local tokens or else it would require having these servers use a token with acl:write to make a
+		// set up the token template - the ids and create
-		// token create RPC to the servers in the primary DC.
+		template := structs.ACLToken{
-		if !c.srv.LocalTokensEnabled() {
-			return fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter: %s", c.srv.config.Datacenter)
-		}
-
-		// generate the accessor id
-		accessor, err := lib.GenerateUUID(c.srv.checkTokenUUID)
-		if err != nil {
-			return err
-		}
-		// generate the secret id
-		secret, err := lib.GenerateUUID(c.srv.checkTokenUUID)
-		if err != nil {
-			return err
-		}
-
-		// set up the token
-		token := structs.ACLToken{
-			AccessorID:  accessor,
-			SecretID:    secret,
 			Description: fmt.Sprintf("Auto Config Token for Node %q", opts.NodeName),
-			CreateTime:  time.Now(),
 			Local:       true,
 			NodeIdentities: []*structs.ACLNodeIdentity{
 				{
 					NodeName:   opts.NodeName,
-					Datacenter: c.srv.config.Datacenter,
+					Datacenter: backendConf.Datacenter,
 				},
 			},
 			EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
 		}
 
-		req := structs.ACLTokenBatchSetRequest{
+		token, err := ac.backend.CreateACLToken(&template)
-			Tokens: structs.ACLTokens{&token},
+		if err != nil {
-			CAS:    false,
+			return fmt.Errorf("Failed to generate an ACL token for node %q - %w", opts.NodeName, err)
 		}
 
-		// perform the request to mint the new token
+		// req := structs.ACLTokenBatchSetRequest{
-		if _, err := c.srv.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
+		// 	Tokens: structs.ACLTokens{&token},
-			return err
+		// 	CAS:    false,
-		}
+		// }
 
-		acl.Tokens = &config.ACLTokens{Agent: secret}
+		// // perform the request to mint the new token
+		// if _, err := c.srv.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
+		// 	return err
+		// }
+
+		acl.Tokens = &config.ACLTokens{Agent: token.SecretID}
 	}
 
 	conf.ACL = acl
@@ -173,19 +167,19 @@ func (c *Cluster) updateACLsInConfig(opts AutoConfigOptions, conf *config.Config
 
 // updateJoinAddressesInConfig determines the correct gossip endpoints that clients should
 // be connecting to for joining the cluster based on the segment given in the opts parameter.
-func (c *Cluster) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *config.Config) error {
+func (ac *AutoConfig) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *config.Config) error {
-	members, err := c.srv.LANSegmentMembers(opts.SegmentName)
+	joinAddrs, err := ac.backend.DatacenterJoinAddresses(opts.SegmentName)
 	if err != nil {
 		return err
 	}
 
-	var joinAddrs []string
+	// var joinAddrs []string
-	for _, m := range members {
+	// for _, m := range members {
-		if ok, _ := metadata.IsConsulServer(m); ok {
+	// 	if ok, _ := metadata.IsConsulServer(m); ok {
-			serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
+	// 		serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
-			joinAddrs = append(joinAddrs, serfAddr.String())
+	// 		joinAddrs = append(joinAddrs, serfAddr.String())
-		}
+	// 	}
-	}
+	// }
 
 	if conf.Gossip == nil {
 		conf.Gossip = &config.Gossip{}
@@ -196,9 +190,9 @@ func (c *Cluster) updateJoinAddressesInConfig(opts AutoConfigOptions, conf *conf
 }
 
 // updateGossipEncryptionInConfig will populate the gossip encryption configuration settings
-func (c *Cluster) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *config.Config) error {
+func (ac *AutoConfig) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *config.Config) error {
 	// Add gossip encryption settings if there is any key loaded
-	memberlistConfig := c.srv.config.SerfLANConfig.MemberlistConfig
+	memberlistConfig := ac.getBackendConfig().SerfLANConfig.MemberlistConfig
 	if lanKeyring := memberlistConfig.Keyring; lanKeyring != nil {
 		if conf.Gossip == nil {
 			conf.Gossip = &config.Gossip{}
@@ -221,13 +215,20 @@ func (c *Cluster) updateGossipEncryptionInConfig(_ AutoConfigOptions, conf *conf
 
 // updateTLSSettingsInConfig will populate the TLS configuration settings but will not
 // populate leaf or ca certficiates.
-func (c *Cluster) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Config) error {
+func (ac *AutoConfig) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Config) error {
+	tlsConfigurator := ac.backend.TLSConfigurator()
+	if tlsConfigurator == nil {
+		// TLS is not enabled?
+		return nil
+	}
+
 	// add in TLS configuration
 	if conf.TLS == nil {
 		conf.TLS = &config.TLS{}
 	}
-	conf.TLS.VerifyServerHostname = c.srv.tlsConfigurator.VerifyServerHostname()
+
-	base := c.srv.tlsConfigurator.Base()
+	conf.TLS.VerifyServerHostname = tlsConfigurator.VerifyServerHostname()
+	base := tlsConfigurator.Base()
 	conf.TLS.VerifyOutgoing = base.VerifyOutgoing
 	conf.TLS.MinVersion = base.TLSMinVersion
 	conf.TLS.PreferServerCipherSuites = base.PreferServerCipherSuites
@@ -239,61 +240,65 @@ func (c *Cluster) updateTLSSettingsInConfig(_ AutoConfigOptions, conf *config.Co
 
 // baseConfig will populate the configuration with some base settings such as the
 // datacenter names, node name etc.
-func (c *Cluster) baseConfig(opts AutoConfigOptions, conf *config.Config) error {
+func (ac *AutoConfig) baseConfig(opts AutoConfigOptions, conf *config.Config) error {
+	backendConf := ac.getBackendConfig()
+
 	if opts.NodeName == "" {
 		return fmt.Errorf("Cannot generate auto config response without a node name")
 	}
 
-	conf.Datacenter = c.srv.config.Datacenter
+	conf.Datacenter = backendConf.Datacenter
-	conf.PrimaryDatacenter = c.srv.config.PrimaryDatacenter
+	conf.PrimaryDatacenter = backendConf.PrimaryDatacenter
 	conf.NodeName = opts.NodeName
 	conf.SegmentName = opts.SegmentName
 
 	return nil
 }
 
-type autoConfigUpdater func(c *Cluster, opts AutoConfigOptions, conf *config.Config) error
+type autoConfigUpdater func(c *AutoConfig, opts AutoConfigOptions, conf *config.Config) error
 
 var (
 	// variable holding the list of config updating functions to execute when generating
 	// the auto config response. This will allow for more easily adding extra self-contained
 	// configurators here in the future.
 	autoConfigUpdaters []autoConfigUpdater = []autoConfigUpdater{
-		(*Cluster).baseConfig,
+		(*AutoConfig).baseConfig,
-		(*Cluster).updateJoinAddressesInConfig,
+		(*AutoConfig).updateJoinAddressesInConfig,
-		(*Cluster).updateGossipEncryptionInConfig,
+		(*AutoConfig).updateGossipEncryptionInConfig,
-		(*Cluster).updateTLSSettingsInConfig,
+		(*AutoConfig).updateTLSSettingsInConfig,
-		(*Cluster).updateACLsInConfig,
+		(*AutoConfig).updateACLsInConfig,
-		(*Cluster).updateTLSCertificatesInConfig,
+		(*AutoConfig).updateTLSCertificatesInConfig,
 	}
 )
 
 // AgentAutoConfig will authorize the incoming request and then generate the configuration
 // to push down to the client
-func (c *Cluster) AutoConfig(req *agentpb.AutoConfigRequest, resp *agentpb.AutoConfigResponse) error {
+func (ac *AutoConfig) InitialConfiguration(req *agentpb.AutoConfigRequest, resp *agentpb.AutoConfigResponse) error {
+	backendConf := ac.getBackendConfig()
+
 	// default the datacenter to our datacenter - agents do not have to specify this as they may not
 	// yet know the datacenter name they are going to be in.
 	if req.Datacenter == "" {
-		req.Datacenter = c.srv.config.Datacenter
+		req.Datacenter = backendConf.Datacenter
 	}
 
 	// TODO (autoconf) Is performing auto configuration over the WAN really a bad idea?
-	if req.Datacenter != c.srv.config.Datacenter {
+	if req.Datacenter != backendConf.Datacenter {
 		return fmt.Errorf("invalid datacenter %q - agent auto configuration cannot target a remote datacenter", req.Datacenter)
 	}
 
 	// forward to the leader
-	if done, err := c.srv.forward("Cluster.AutoConfig", req, req, resp); done {
+	if done, err := ac.backend.ForwardRPC("AutoConfig.InitialConfiguration", req, req, resp); done {
 		return err
 	}
 
 	// TODO (autoconf) maybe panic instead?
-	if c.authorizer == nil {
+	if ac.authorizer == nil {
 		return fmt.Errorf("No Auto Config authorizer is configured")
 	}
 
 	// authorize the request with the configured authorizer
-	opts, err := c.authorizer.Authorize(req)
+	opts, err := ac.authorizer.Authorize(req)
 	if err != nil {
 		return err
 	}
@@ -302,7 +307,7 @@ func (c *Cluster) AutoConfig(req *agentpb.AutoConfigRequest, resp *agentpb.AutoC
 
 	// update all the configurations
 	for _, configFn := range autoConfigUpdaters {
-		if err := configFn(c, opts, conf); err != nil {
+		if err := configFn(ac, opts, conf); err != nil {
 			return err
 		}
 	}

agent/consul/auto_config_endpoint_test.go

@@ -19,11 +19,49 @@ import (
 	"github.com/hashicorp/consul/tlsutil"
 	"github.com/hashicorp/memberlist"
 	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
 	"gopkg.in/square/go-jose.v2/jwt"
 )
 
+type mockAutoConfigBackend struct {
+	mock.Mock
+}
+
+func (m *mockAutoConfigBackend) GetConfig() *Config {
+	ret := m.Called()
+	// this handles converting an untyped nil to a typed nil
+	cfg, _ := ret.Get(0).(*Config)
+	return cfg
+}
+
+func (m *mockAutoConfigBackend) CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error) {
+	ret := m.Called(template)
+	// this handles converting an untyped nil to a typed nil
+	token, _ := ret.Get(0).(*structs.ACLToken)
+	return token, ret.Error(1)
+}
+
+func (m *mockAutoConfigBackend) DatacenterJoinAddresses(segment string) ([]string, error) {
+	ret := m.Called(segment)
+	// this handles converting an untyped nil to a typed nil
+	addrs, _ := ret.Get(0).([]string)
+	return addrs, ret.Error(1)
+}
+
+func (m *mockAutoConfigBackend) TLSConfigurator() *tlsutil.Configurator {
+	ret := m.Called()
+	// this handles converting an untyped nil to a typed nil
+	cfg, _ := ret.Get(0).(*tlsutil.Configurator)
+	return cfg
+}
+
+func (m *mockAutoConfigBackend) ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error) {
+	ret := m.Called(method, info, args, reply)
+	return ret.Bool(0), ret.Error(1)
+}
+
 func testJWTStandardClaims() jwt.Claims {
 	now := time.Now()
 
@@ -48,13 +86,13 @@ func signJWTWithStandardClaims(t *testing.T, privKey string, claims interface{})
 	return signJWT(t, privKey, testJWTStandardClaims(), claims)
 }
 
-// TestClusterAutoConfig is really an integration test of all the moving parts of the Cluster.AutoConfig RPC.
+// TestAutoConfigInitialConfiguration is really an integration test of all the moving parts of the AutoConfig.InitialConfiguration RPC.
 // Full testing of the individual parts will not be done in this test:
 //
 //  * Any implementations of the AutoConfigAuthorizer interface (although these test do use the jwtAuthorizer)
-//  * Each of the individual config generation functions. These can be unit tested separately and many wont
+//  * Each of the individual config generation functions. These can be unit tested separately and should NOT
-//    require a running test server.
+//    require running test servers
-func TestClusterAutoConfig(t *testing.T) {
+func TestAutoConfigInitialConfiguration(t *testing.T) {
 	type testCase struct {
 		request       agentpb.AutoConfigRequest
 		expected      agentpb.AutoConfigResponse
@@ -227,7 +265,7 @@ func TestClusterAutoConfig(t *testing.T) {
 	for testName, tcase := range cases {
 		t.Run(testName, func(t *testing.T) {
 			var reply agentpb.AutoConfigResponse
-			err := msgpackrpc.CallWithCodec(codec, "Cluster.AutoConfig", &tcase.request, &reply)
+			err := msgpackrpc.CallWithCodec(codec, "AutoConfig.InitialConfiguration", &tcase.request, &reply)
 			if tcase.err != "" {
 				testutil.RequireErrorContains(t, err, tcase.err)
 			} else {
@@ -241,7 +279,7 @@ func TestClusterAutoConfig(t *testing.T) {
 	}
 }
 
-func TestClusterAutoConfig_baseConfig(t *testing.T) {
+func TestAutoConfig_baseConfig(t *testing.T) {
 	type testCase struct {
 		serverConfig Config
 		opts         AutoConfigOptions
@@ -277,25 +315,28 @@ func TestClusterAutoConfig_baseConfig(t *testing.T) {
 
 	for name, tcase := range cases {
 		t.Run(name, func(t *testing.T) {
-			cluster := Cluster{
+			backend := &mockAutoConfigBackend{}
-				srv: &Server{
+			backend.On("GetConfig").Return(&tcase.serverConfig).Once()
-					config: &tcase.serverConfig,
+
-				},
+			ac := AutoConfig{
+				backend: backend,
 			}
 
 			var actual config.Config
-			err := cluster.baseConfig(tcase.opts, &actual)
+			err := ac.baseConfig(tcase.opts, &actual)
 			if tcase.err == "" {
 				require.NoError(t, err)
 				require.Equal(t, tcase.expected, actual)
 			} else {
 				testutil.RequireErrorContains(t, err, tcase.err)
 			}
+
+			backend.AssertExpectations(t)
 		})
 	}
 }
 
-func TestClusterAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
+func TestAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
 	_, _, cacert, err := testTLSCertificates("server.dc1.consul")
 	require.NoError(t, err)
 
@@ -365,16 +406,19 @@ func TestClusterAutoConfig_updateTLSSettingsInConfig(t *testing.T) {
 			configurator, err := tlsutil.NewConfigurator(tcase.tlsConfig, logger)
 			require.NoError(t, err)
 
-			cluster := &Cluster{
+			backend := &mockAutoConfigBackend{}
-				srv: &Server{
+			backend.On("TLSConfigurator").Return(configurator).Once()
-					tlsConfigurator: configurator,
+
-				},
+			ac := &AutoConfig{
+				backend: backend,
 			}
 
 			var actual config.Config
-			err = cluster.updateTLSSettingsInConfig(AutoConfigOptions{}, &actual)
+			err = ac.updateTLSSettingsInConfig(AutoConfigOptions{}, &actual)
 			require.NoError(t, err)
 			require.Equal(t, tcase.expected, actual)
+
+			backend.AssertExpectations(t)
 		})
 	}
 }
@@ -436,18 +480,22 @@ func TestAutoConfig_updateGossipEncryptionInConfig(t *testing.T) {
 
 	for name, tcase := range cases {
 		t.Run(name, func(t *testing.T) {
-			cluster := Cluster{
+			cfg := DefaultConfig()
-				srv: &Server{
+			cfg.SerfLANConfig.MemberlistConfig = &tcase.conf
-					config: DefaultConfig(),
+
-				},
+			backend := &mockAutoConfigBackend{}
+			backend.On("GetConfig").Return(cfg).Once()
+
+			ac := AutoConfig{
+				backend: backend,
 			}
 
-			cluster.srv.config.SerfLANConfig.MemberlistConfig = &tcase.conf
-
 			var actual config.Config
-			err := cluster.updateGossipEncryptionInConfig(AutoConfigOptions{}, &actual)
+			err := ac.updateGossipEncryptionInConfig(AutoConfigOptions{}, &actual)
 			require.NoError(t, err)
 			require.Equal(t, tcase.expected, actual)
+
+			backend.AssertExpectations(t)
 		})
 	}
 }
@@ -481,40 +529,53 @@ func TestAutoConfig_updateTLSCertificatesInConfig(t *testing.T) {
 
 	for name, tcase := range cases {
 		t.Run(name, func(t *testing.T) {
-			cluster := Cluster{
+			backend := &mockAutoConfigBackend{}
-				srv: &Server{
+			backend.On("GetConfig").Return(&tcase.serverConfig).Once()
-					config: &tcase.serverConfig,
+
-				},
+			ac := AutoConfig{
+				backend: backend,
 			}
 
 			var actual config.Config
-			err := cluster.updateTLSCertificatesInConfig(AutoConfigOptions{}, &actual)
+			err := ac.updateTLSCertificatesInConfig(AutoConfigOptions{}, &actual)
 			require.NoError(t, err)
 			require.Equal(t, tcase.expected, actual)
+
+			backend.AssertExpectations(t)
 		})
 	}
 }
 
 func TestAutoConfig_updateACLsInConfig(t *testing.T) {
 	type testCase struct {
-		patch    func(c *Config)
+		config         Config
-		expected config.Config
+		expected       config.Config
-		verify   func(t *testing.T, c *config.Config)
+		expectACLToken bool
-		err      string
+		err            error
 	}
 
+	const (
+		tokenAccessor = "b98761aa-c0ee-445b-9b0c-f54b56b47778"
+		tokenSecret   = "1c96448a-ab04-4caa-982a-e8b095a111e2"
+	)
+
+	testDC := "dc1"
+
 	cases := map[string]testCase{
 		"enabled": {
-			patch: func(c *Config) {
+			config: Config{
-				c.ACLsEnabled = true
+				Datacenter:             testDC,
-				c.ACLPolicyTTL = 7 * time.Second
+				PrimaryDatacenter:      testDC,
-				c.ACLRoleTTL = 10 * time.Second
+				ACLsEnabled:            true,
-				c.ACLTokenTTL = 12 * time.Second
+				ACLPolicyTTL:           7 * time.Second,
-				c.ACLDisabledTTL = 31 * time.Second
+				ACLRoleTTL:             10 * time.Second,
-				c.ACLDefaultPolicy = "allow"
+				ACLTokenTTL:            12 * time.Second,
-				c.ACLDownPolicy = "deny"
+				ACLDisabledTTL:         31 * time.Second,
-				c.ACLEnableKeyListPolicy = true
+				ACLDefaultPolicy:       "allow",
+				ACLDownPolicy:          "deny",
+				ACLEnableKeyListPolicy: true,
 			},
+			expectACLToken: true,
 			expected: config.Config{
 				ACL: &config.ACL{
 					Enabled:             true,
@@ -525,32 +586,26 @@ func TestAutoConfig_updateACLsInConfig(t *testing.T) {
 					DownPolicy:          "deny",
 					DefaultPolicy:       "allow",
 					EnableKeyListPolicy: true,
-					Tokens:              &config.ACLTokens{Agent: "verified"},
+					Tokens: &config.ACLTokens{
+						Agent: tokenSecret,
+					},
 				},
 			},
-			verify: func(t *testing.T, c *config.Config) {
-				t.Helper()
-				// the agent token secret is non-deterministically generated
-				// So we want to validate that one was set and overwrite with
-				// a value that the expected configurate wants.
-				require.NotNil(t, c)
-				require.NotNil(t, c.ACL)
-				require.NotNil(t, c.ACL.Tokens)
-				require.NotEmpty(t, c.ACL.Tokens.Agent)
-				c.ACL.Tokens.Agent = "verified"
-			},
 		},
 		"disabled": {
-			patch: func(c *Config) {
+			config: Config{
-				c.ACLsEnabled = false
+				Datacenter:             testDC,
-				c.ACLPolicyTTL = 7 * time.Second
+				PrimaryDatacenter:      testDC,
-				c.ACLRoleTTL = 10 * time.Second
+				ACLsEnabled:            false,
-				c.ACLTokenTTL = 12 * time.Second
+				ACLPolicyTTL:           7 * time.Second,
-				c.ACLDisabledTTL = 31 * time.Second
+				ACLRoleTTL:             10 * time.Second,
-				c.ACLDefaultPolicy = "allow"
+				ACLTokenTTL:            12 * time.Second,
-				c.ACLDownPolicy = "deny"
+				ACLDisabledTTL:         31 * time.Second,
-				c.ACLEnableKeyListPolicy = true
+				ACLDefaultPolicy:       "allow",
+				ACLDownPolicy:          "deny",
+				ACLEnableKeyListPolicy: true,
 			},
+			expectACLToken: false,
 			expected: config.Config{
 				ACL: &config.ACL{
 					Enabled:             false,
@@ -565,53 +620,79 @@ func TestAutoConfig_updateACLsInConfig(t *testing.T) {
 			},
 		},
 		"local-tokens-disabled": {
-			patch: func(c *Config) {
+			config: Config{
-				c.PrimaryDatacenter = "somewhere else"
+				Datacenter:        testDC,
+				PrimaryDatacenter: "somewhere-else",
+				ACLsEnabled:       true,
 			},
-			err: "Agent Auto Configuration requires local token usage to be enabled in this datacenter",
+			expectACLToken: true,
+			err:            fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter"),
 		},
 	}
 	for name, tcase := range cases {
 		t.Run(name, func(t *testing.T) {
-			_, s, _ := testACLServerWithConfig(t, tcase.patch, false)
+			backend := &mockAutoConfigBackend{}
+			backend.On("GetConfig").Return(&tcase.config).Once()
 
-			waitForLeaderEstablishment(t, s)
+			expectedTemplate := &structs.ACLToken{
+				Description: `Auto Config Token for Node "something"`,
+				Local:       true,
+				NodeIdentities: []*structs.ACLNodeIdentity{
+					{
+						NodeName:   "something",
+						Datacenter: testDC,
+					},
+				},
+				EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
+			}
 
-			cluster := Cluster{srv: s}
+			testToken := &structs.ACLToken{
+				AccessorID:  tokenAccessor,
+				SecretID:    tokenSecret,
+				Description: `Auto Config Token for Node "something"`,
+				Local:       true,
+				NodeIdentities: []*structs.ACLNodeIdentity{
+					{
+						NodeName:   "something",
+						Datacenter: testDC,
+					},
+				},
+				EnterpriseMeta: *structs.DefaultEnterpriseMeta(),
+			}
+
+			if tcase.expectACLToken {
+				backend.On("CreateACLToken", expectedTemplate).Return(testToken, tcase.err).Once()
+			}
+
+			ac := AutoConfig{backend: backend}
 
 			var actual config.Config
-			err := cluster.updateACLsInConfig(AutoConfigOptions{NodeName: "something"}, &actual)
+			err := ac.updateACLsInConfig(AutoConfigOptions{NodeName: "something"}, &actual)
-			if tcase.err != "" {
+			if tcase.err != nil {
-				testutil.RequireErrorContains(t, err, tcase.err)
+				testutil.RequireErrorContains(t, err, tcase.err.Error())
 			} else {
 				require.NoError(t, err)
-				if tcase.verify != nil {
-					tcase.verify(t, &actual)
-				}
 				require.Equal(t, tcase.expected, actual)
 			}
+
+			backend.AssertExpectations(t)
 		})
 	}
 }
 
 func TestAutoConfig_updateJoinAddressesInConfig(t *testing.T) {
-	conf := testClusterConfig{
+	addrs := []string{"198.18.0.7:8300", "198.18.0.1:8300"}
-		Datacenter: "primary",
+	backend := &mockAutoConfigBackend{}
-		Servers:    3,
+	backend.On("DatacenterJoinAddresses", "").Return(addrs, nil).Once()
-	}
 
-	nodes := newTestCluster(t, &conf)
+	ac := AutoConfig{backend: backend}
-
-	cluster := Cluster{srv: nodes.Servers[0]}
 
 	var actual config.Config
-	err := cluster.updateJoinAddressesInConfig(AutoConfigOptions{}, &actual)
+	err := ac.updateJoinAddressesInConfig(AutoConfigOptions{}, &actual)
 	require.NoError(t, err)
 
-	var expected []string
-	for _, srv := range nodes.Servers {
-		expected = append(expected, fmt.Sprintf("127.0.0.1:%d", srv.config.SerfLANConfig.MemberlistConfig.BindPort))
-	}
 	require.NotNil(t, actual.Gossip)
-	require.ElementsMatch(t, expected, actual.Gossip.RetryJoinLAN)
+	require.ElementsMatch(t, addrs, actual.Gossip.RetryJoinLAN)
+
+	backend.AssertExpectations(t)
 }

agent/consul/server.go

@@ -876,7 +876,7 @@ func (s *Server) setupRPC() error {
 		authz = &disabledAuthorizer{}
 	}
 	// now register with the insecure RPC server
-	s.insecureRPCServer.Register(&Cluster{srv: s, authorizer: authz})
+	s.insecureRPCServer.Register(&AutoConfig{backend: s, authorizer: authz})
 
 	ln, err := net.ListenTCP("tcp", s.config.RPCAddr)
 	if err != nil {
@@ -1436,6 +1436,89 @@ func (s *Server) intentionReplicationEnabled() bool {
 	return s.config.ConnectEnabled && s.config.Datacenter != s.config.PrimaryDatacenter
 }
 
+// CreateACLToken will create an ACL token from the given template
+func (s *Server) CreateACLToken(template *structs.ACLToken) (*structs.ACLToken, error) {
+	// we have to require local tokens or else it would require having these servers use a token with acl:write to make a
+	// token create RPC to the servers in the primary DC.
+	if !s.LocalTokensEnabled() {
+		return nil, fmt.Errorf("Agent Auto Configuration requires local token usage to be enabled in this datacenter: %s", s.config.Datacenter)
+	}
+
+	newToken := *template
+
+	// generate the accessor id
+	if newToken.AccessorID == "" {
+		accessor, err := lib.GenerateUUID(s.checkTokenUUID)
+		if err != nil {
+			return nil, err
+		}
+
+		newToken.AccessorID = accessor
+	}
+
+	// generate the secret id
+	if newToken.SecretID == "" {
+		secret, err := lib.GenerateUUID(s.checkTokenUUID)
+		if err != nil {
+			return nil, err
+		}
+
+		newToken.SecretID = secret
+	}
+
+	newToken.CreateTime = time.Now()
+
+	req := structs.ACLTokenBatchSetRequest{
+		Tokens: structs.ACLTokens{&newToken},
+		CAS:    false,
+	}
+
+	// perform the request to mint the new token
+	if _, err := s.raftApplyMsgpack(structs.ACLTokenSetRequestType, &req); err != nil {
+		return nil, err
+	}
+
+	// return the full token definition from the FSM
+	_, token, err := s.fsm.State().ACLTokenGetByAccessor(nil, newToken.AccessorID, &newToken.EnterpriseMeta)
+	return token, err
+}
+
+// DatacenterJoinAddresses will return all the strings suitable for usage in
+// retry join operations to connect to the the LAN or LAN segment gossip pool.
+func (s *Server) DatacenterJoinAddresses(segment string) ([]string, error) {
+	members, err := s.LANSegmentMembers(segment)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to retrieve members for segment %s - %w", segment, err)
+	}
+
+	var joinAddrs []string
+	for _, m := range members {
+		if ok, _ := metadata.IsConsulServer(m); ok {
+			serfAddr := net.TCPAddr{IP: m.Addr, Port: int(m.Port)}
+			joinAddrs = append(joinAddrs, serfAddr.String())
+		}
+	}
+
+	return joinAddrs, nil
+}
+
+// ForwardRPC is basically just renaming forward, in a future commit I am going to actually do the rename.
+func (s *Server) ForwardRPC(method string, info structs.RPCInfo, args, reply interface{}) (bool, error) {
+	return s.forward(method, info, args, reply)
+}
+
+// GetConfig will return the servers configuration - this is needed to satisfy
+// interfaces: AutoConfigDelegate
+func (s *Server) GetConfig() *Config {
+	return s.config
+}
+
+// TLSConfigurator just returns the servers TLS Configurator - this is needed to satisfy
+// interfaces: AutoConfigDelegate
+func (s *Server) TLSConfigurator() *tlsutil.Configurator {
+	return s.tlsConfigurator
+}
+
 // peersInfoContent is used to help operators understand what happened to the
 // peers.json file. This is written to a file called peers.info in the same
 // location.

agent/consul/server_test.go

@@ -1534,3 +1534,97 @@ func TestServer_CALogging(t *testing.T) {
 
 	require.Contains(t, buf.String(), "consul CA provider configured")
 }
+
+func TestServer_DatacenterJoinAddresses(t *testing.T) {
+	conf := testClusterConfig{
+		Datacenter: "primary",
+		Servers:    3,
+	}
+
+	nodes := newTestCluster(t, &conf)
+
+	var expected []string
+	for _, srv := range nodes.Servers {
+		expected = append(expected, fmt.Sprintf("127.0.0.1:%d", srv.config.SerfLANConfig.MemberlistConfig.BindPort))
+	}
+
+	actual, err := nodes.Servers[0].DatacenterJoinAddresses("")
+	require.NoError(t, err)
+	require.ElementsMatch(t, expected, actual)
+}
+
+func TestServer_CreateACLToken(t *testing.T) {
+	_, srv, codec := testACLServerWithConfig(t, nil, false)
+
+	waitForLeaderEstablishment(t, srv)
+
+	r1, err := upsertTestRole(codec, TestDefaultMasterToken, "dc1")
+	require.NoError(t, err)
+
+	t.Run("predefined-ids", func(t *testing.T) {
+		accessor := "554cd3ab-5d4e-4d6e-952e-4e8b6c77bfb3"
+		secret := "ef453f31-ad58-4ec8-8bf8-342e99763026"
+		in := &structs.ACLToken{
+			AccessorID:  accessor,
+			SecretID:    secret,
+			Description: "test",
+			Policies: []structs.ACLTokenPolicyLink{
+				{
+					ID: structs.ACLPolicyGlobalManagementID,
+				},
+			},
+			NodeIdentities: []*structs.ACLNodeIdentity{
+				{
+					NodeName:   "foo",
+					Datacenter: "bar",
+				},
+			},
+			ServiceIdentities: []*structs.ACLServiceIdentity{
+				{
+					ServiceName: "web",
+				},
+			},
+			Roles: []structs.ACLTokenRoleLink{
+				{
+					ID: r1.ID,
+				},
+			},
+		}
+
+		out, err := srv.CreateACLToken(in)
+		require.NoError(t, err)
+		require.Equal(t, accessor, out.AccessorID)
+		require.Equal(t, secret, out.SecretID)
+		require.Equal(t, "test", out.Description)
+		require.NotZero(t, out.CreateTime)
+		require.Len(t, out.Policies, 1)
+		require.Len(t, out.Roles, 1)
+		require.Len(t, out.NodeIdentities, 1)
+		require.Len(t, out.ServiceIdentities, 1)
+		require.Equal(t, structs.ACLPolicyGlobalManagementID, out.Policies[0].ID)
+		require.Equal(t, "foo", out.NodeIdentities[0].NodeName)
+		require.Equal(t, "web", out.ServiceIdentities[0].ServiceName)
+		require.Equal(t, r1.ID, out.Roles[0].ID)
+	})
+
+	t.Run("autogen-ids", func(t *testing.T) {
+		in := &structs.ACLToken{
+			Description: "test",
+			NodeIdentities: []*structs.ACLNodeIdentity{
+				{
+					NodeName:   "foo",
+					Datacenter: "bar",
+				},
+			},
+		}
+
+		out, err := srv.CreateACLToken(in)
+		require.NoError(t, err)
+		require.NotEmpty(t, out.AccessorID)
+		require.NotEmpty(t, out.SecretID)
+		require.Equal(t, "test", out.Description)
+		require.NotZero(t, out.CreateTime)
+		require.Len(t, out.NodeIdentities, 1)
+		require.Equal(t, "foo", out.NodeIdentities[0].NodeName)
+	})
+}

agent/pool/pool.go

@@ -553,7 +553,7 @@ func (p *ConnPool) RPC(
 	// secure or insecure variant depending on whether its an ongoing
 	// or first time config request. For now though this is fine until
 	// those ongoing requests are implemented.
-	if method == "AutoEncrypt.Sign" || method == "Cluster.AutoConfig" {
+	if method == "AutoEncrypt.Sign" || method == "AutoConfig.InitialConfiguration" {
 		return p.rpcInsecure(dc, addr, method, args, reply)
 	} else {
 		return p.rpc(dc, nodeName, addr, method, args, reply)