Fix peering acceptors in secondary datacenters. (#16233)

Prior to this commit, secondary datacenters could not be initialized
as peering acceptors if ACLs were enabled. This is due to the fact that
internal server-to-server API calls would fail because the management
token was not generated. This PR makes it so that both primary and
secondary datacenters generate their own management token whenever
a leader is elected in their respective clusters.

.changelog/16230.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+peering: Fix issue where secondary wan-federated datacenters could not be used as peering acceptors.
+```

agent/consul/leader.go

@@ -529,25 +529,25 @@ func (s *Server) initializeACLs(ctx context.Context) error {
 			s.logger.Info("Created ACL anonymous token from configuration")
 		}
 
-		// Generate or rotate the server management token on leadership transitions.
-		// This token is used by Consul servers for authn/authz when making
-		// requests to themselves through public APIs such as the agent cache.
-		// It is stored as system metadata because it is internally
-		// managed and users are not meant to see it or interact with it.
-		secretID, err := lib.GenerateUUID(nil)
-		if err != nil {
-			return fmt.Errorf("failed to generate the secret ID for the server management token: %w", err)
-		}
-		if err := s.setSystemMetadataKey(structs.ServerManagementTokenAccessorID, secretID); err != nil {
-			return fmt.Errorf("failed to persist server management token: %w", err)
-		}
-
 		// launch the upgrade go routine to generate accessors for everything
 		s.startACLUpgrade(ctx)
 	} else {
 		s.startACLReplication(ctx)
 	}
 
+	// Generate or rotate the server management token on leadership transitions.
+	// This token is used by Consul servers for authn/authz when making
+	// requests to themselves through public APIs such as the agent cache.
+	// It is stored as system metadata because it is internally
+	// managed and users are not meant to see it or interact with it.
+	secretID, err := lib.GenerateUUID(nil)
+	if err != nil {
+		return fmt.Errorf("failed to generate the secret ID for the server management token: %w", err)
+	}
+	if err := s.setSystemMetadataKey(structs.ServerManagementTokenAccessorID, secretID); err != nil {
+		return fmt.Errorf("failed to persist server management token: %w", err)
+	}
+
 	s.startACLTokenReaping(ctx)
 
 	return nil

agent/consul/leader_test.go

@@ -1308,6 +1308,51 @@ func TestLeader_ACL_Initialization(t *testing.T) {
 	}
 }
 
+func TestLeader_ACL_Initialization_SecondaryDC(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
+	t.Parallel()
+
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.Bootstrap = true
+		c.Datacenter = "dc1"
+		c.PrimaryDatacenter = "dc1"
+		c.ACLsEnabled = true
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
+
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.Bootstrap = true
+		c.Datacenter = "dc2"
+		c.PrimaryDatacenter = "dc1"
+		c.ACLsEnabled = true
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+	testrpc.WaitForTestAgent(t, s2.RPC, "dc2")
+
+	// Check dc1's management token
+	serverToken1, err := s1.getSystemMetadata(structs.ServerManagementTokenAccessorID)
+	require.NoError(t, err)
+	require.NotEmpty(t, serverToken1)
+	_, err = uuid.ParseUUID(serverToken1)
+	require.NoError(t, err)
+
+	// Check dc2's management token
+	serverToken2, err := s2.getSystemMetadata(structs.ServerManagementTokenAccessorID)
+	require.NoError(t, err)
+	require.NotEmpty(t, serverToken2)
+	_, err = uuid.ParseUUID(serverToken2)
+	require.NoError(t, err)
+
+	// Ensure the tokens were not replicated between clusters.
+	require.NotEqual(t, serverToken1, serverToken2)
+}
+
 func TestLeader_ACLUpgrade_IsStickyEvenIfSerfTagsRegress(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")