Fix peering acceptors in secondary datacenters. (#16233)

Prior to this commit, secondary datacenters could not be initialized
as peering acceptors if ACLs were enabled. This is due to the fact that
internal server-to-server API calls would fail because the management
token was not generated. This PR makes it so that both primary and
secondary datacenters generate their own management token whenever
a leader is elected in their respective clusters.

.changelog/16230.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+peering: Fix issue where secondary wan-federated datacenters could not be used as peering acceptors.
+```

agent/consul/leader.go

@@ -529,6 +529,12 @@ func (s *Server) initializeACLs(ctx context.Context) error {
 			s.logger.Info("Created ACL anonymous token from configuration")
 		}
 
+		// launch the upgrade go routine to generate accessors for everything
+		s.startACLUpgrade(ctx)
+	} else {
+		s.startACLReplication(ctx)
+	}
+
 	// Generate or rotate the server management token on leadership transitions.
 	// This token is used by Consul servers for authn/authz when making
 	// requests to themselves through public APIs such as the agent cache.
@@ -542,12 +548,6 @@ func (s *Server) initializeACLs(ctx context.Context) error {
 		return fmt.Errorf("failed to persist server management token: %w", err)
 	}
 
-		// launch the upgrade go routine to generate accessors for everything
-		s.startACLUpgrade(ctx)
-	} else {
-		s.startACLReplication(ctx)
-	}
-
 	s.startACLTokenReaping(ctx)
 
 	return nil

agent/consul/leader_test.go

@@ -1308,6 +1308,51 @@ func TestLeader_ACL_Initialization(t *testing.T) {
 	}
 }
 
+func TestLeader_ACL_Initialization_SecondaryDC(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
+	t.Parallel()
+
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.Bootstrap = true
+		c.Datacenter = "dc1"
+		c.PrimaryDatacenter = "dc1"
+		c.ACLsEnabled = true
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
+
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.Bootstrap = true
+		c.Datacenter = "dc2"
+		c.PrimaryDatacenter = "dc1"
+		c.ACLsEnabled = true
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+	testrpc.WaitForTestAgent(t, s2.RPC, "dc2")
+
+	// Check dc1's management token
+	serverToken1, err := s1.getSystemMetadata(structs.ServerManagementTokenAccessorID)
+	require.NoError(t, err)
+	require.NotEmpty(t, serverToken1)
+	_, err = uuid.ParseUUID(serverToken1)
+	require.NoError(t, err)
+
+	// Check dc2's management token
+	serverToken2, err := s2.getSystemMetadata(structs.ServerManagementTokenAccessorID)
+	require.NoError(t, err)
+	require.NotEmpty(t, serverToken2)
+	_, err = uuid.ParseUUID(serverToken2)
+	require.NoError(t, err)
+
+	// Ensure the tokens were not replicated between clusters.
+	require.NotEqual(t, serverToken1, serverToken2)
+}
+
 func TestLeader_ACLUpgrade_IsStickyEvenIfSerfTagsRegress(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")