consul/test-integ/peering_commontopo/ac6_failovers_test.go

package peering

import (
	"fmt"
	"testing"

	"github.com/hashicorp/consul/testing/deployer/topology"
	"github.com/stretchr/testify/require"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
)

// note: unlike other *Suite structs that are per-peering direction,
// this one is special and does all directions itself, because the
// setup is not exactly symmetrical
type ac6FailoversSuite struct {
	ac6 map[nodeKey]ac6FailoversContext
}
type ac6FailoversContext struct {
	clientSID topology.ServiceID
	serverSID topology.ServiceID

	// used to remove the node and trigger failover
	serverNode topology.NodeID
}
type nodeKey struct {
	dc        string
	partition string
}

// Note: this test cannot share topo
func TestAC6Failovers(t *testing.T) {
	ct := NewCommonTopo(t)
	s := &ac6FailoversSuite{}
	s.setup(t, ct)
	ct.Launch(t)
	s.test(t, ct)
}

func (s *ac6FailoversSuite) setup(t *testing.T, ct *commonTopo) {
	// TODO: update setups to loop through a cluster's partitions+namespaces internally
	s.setupAC6Failovers(ct, ct.DC1, ct.DC2)
	s.setupAC6Failovers(ct, ct.DC2, ct.DC1)
	s.setupAC6FailoversDC3(ct, ct.DC3, ct.DC1, ct.DC2)
}

// dc1 is peered with dc2 and dc3.
// dc1 has an ac6-client in "default" and "part1" partitions (only default in OSS).
// ac6-client has a single upstream ac6-failover-svc in its respective partition^.
//
// ac6-failover-svc has the following failovers:
//   - peer-dc2-default
//   - peer-dc2-part1 (not in OSS)
//   - peer-dc3-default
//
// This setup is mirrored from dc2->dc1 as well
// (both dcs have dc3 as the last failover target)
//
// ^NOTE: There are no cross-partition upstreams because MeshGatewayMode = local
// and failover information gets stripped out by the mesh gateways so we
// can't test failovers.
func (s *ac6FailoversSuite) setupAC6Failovers(ct *commonTopo, clu, peerClu *topology.Cluster) {
	for _, part := range clu.Partitions {
		partition := part.Name

		// There is a peering per partition in the peered cluster
		var peers []string
		for _, peerPart := range peerClu.Partitions {
			peers = append(peers, LocalPeerName(peerClu, peerPart.Name))
		}

		// Make an HTTP server with various failover targets
		serverSID := topology.ServiceID{
			Name:      "ac6-failover-svc",
			Partition: partition,
		}
		server := NewFortioServiceWithDefaults(
			clu.Datacenter,
			serverSID,
			nil,
		)
		// Export to all known peers
		ct.ExportService(clu, partition,
			api.ExportedService{
				Name: server.ID.Name,
				Consumers: func() []api.ServiceConsumer {
					var consumers []api.ServiceConsumer
					for _, peer := range peers {
						consumers = append(consumers, api.ServiceConsumer{
							Peer: peer,
						})
					}
					return consumers
				}(),
			},
		)
		serverNode := ct.AddServiceNode(clu, serviceExt{Service: server})

		clu.InitialConfigEntries = append(clu.InitialConfigEntries,
			&api.ServiceConfigEntry{
				Kind:      api.ServiceDefaults,
				Name:      server.ID.Name,
				Partition: ConfigEntryPartition(partition),
				Protocol:  "http",
			},
			&api.ServiceResolverConfigEntry{
				Kind:      api.ServiceResolver,
				Name:      server.ID.Name,
				Partition: ConfigEntryPartition(partition),
				Failover: map[string]api.ServiceResolverFailover{
					"*": {
						Targets: func() []api.ServiceResolverFailoverTarget {
							// Make a failover target for every partition in the peer cluster
							var targets []api.ServiceResolverFailoverTarget
							for _, peer := range peers {
								targets = append(targets, api.ServiceResolverFailoverTarget{
									Peer: peer,
								})
							}
							// Just hard code default partition for dc3, since the exhaustive
							// testing will be done against dc2.
							targets = append(targets, api.ServiceResolverFailoverTarget{
								Peer: "peer-dc3-default",
							})
							return targets
						}(),
					},
				},
			},
		)

		// Make client which will dial server
		clientSID := topology.ServiceID{
			Name:      "ac6-client",
			Partition: partition,
		}
		client := NewFortioServiceWithDefaults(
			clu.Datacenter,
			clientSID,
			func(s *topology.Service) {
				// Upstream per partition
				s.Upstreams = []*topology.Upstream{
					{
						ID: topology.ServiceID{
							Name:      server.ID.Name,
							Partition: part.Name,
						},
						LocalPort: 5000,
						// exposed so we can hit it directly
						// TODO: we shouldn't do this; it's not realistic
						LocalAddress: "0.0.0.0",
					},
				}
			},
		)
		ct.ExportService(clu, partition,
			api.ExportedService{
				Name: client.ID.Name,
				Consumers: func() []api.ServiceConsumer {
					var consumers []api.ServiceConsumer
					// Export to each peer
					for _, peer := range peers {
						consumers = append(consumers, api.ServiceConsumer{
							Peer: peer,
						})
					}
					return consumers
				}(),
			},
		)
		ct.AddServiceNode(clu, serviceExt{Service: client})

		clu.InitialConfigEntries = append(clu.InitialConfigEntries,
			&api.ServiceConfigEntry{
				Kind:      api.ServiceDefaults,
				Name:      client.ID.Name,
				Partition: ConfigEntryPartition(partition),
				Protocol:  "http",
			},
		)

		// Add intention allowing local and peered clients to call server
		clu.InitialConfigEntries = append(clu.InitialConfigEntries,
			&api.ServiceIntentionsConfigEntry{
				Kind:      api.ServiceIntentions,
				Name:      server.ID.Name,
				Partition: ConfigEntryPartition(partition),
				// SourceIntention for local client and peered clients
				Sources: func() []*api.SourceIntention {
					ixns := []*api.SourceIntention{
						{
							Name:      client.ID.Name,
							Partition: ConfigEntryPartition(part.Name),
							Action:    api.IntentionActionAllow,
						},
					}
					for _, peer := range peers {
						ixns = append(ixns, &api.SourceIntention{
							Name:   client.ID.Name,
							Peer:   peer,
							Action: api.IntentionActionAllow,
						})
					}
					return ixns
				}(),
			},
		)
		if s.ac6 == nil {
			s.ac6 = map[nodeKey]ac6FailoversContext{}
		}
		s.ac6[nodeKey{clu.Datacenter, partition}] = struct {
			clientSID  topology.ServiceID
			serverSID  topology.ServiceID
			serverNode topology.NodeID
		}{
			clientSID:  clientSID,
			serverSID:  serverSID,
			serverNode: serverNode.ID(),
		}
	}
}

func (s *ac6FailoversSuite) setupAC6FailoversDC3(ct *commonTopo, clu, peer1, peer2 *topology.Cluster) {
	var peers []string
	for _, part := range peer1.Partitions {
		peers = append(peers, LocalPeerName(peer1, part.Name))
	}
	for _, part := range peer2.Partitions {
		peers = append(peers, LocalPeerName(peer2, part.Name))
	}

	partition := "default"

	// Make an HTTP server
	server := NewFortioServiceWithDefaults(
		clu.Datacenter,
		topology.ServiceID{
			Name:      "ac6-failover-svc",
			Partition: partition,
		},
		nil,
	)

	ct.AddServiceNode(clu, serviceExt{
		Service: server,
		Config: &api.ServiceConfigEntry{
			Kind:      api.ServiceDefaults,
			Name:      server.ID.Name,
			Partition: ConfigEntryPartition(partition),
			Protocol:  "http",
		},
		Intentions: &api.ServiceIntentionsConfigEntry{
			Kind:      api.ServiceIntentions,
			Name:      server.ID.Name,
			Partition: ConfigEntryPartition(partition),
			Sources: func() []*api.SourceIntention {
				var ixns []*api.SourceIntention
				for _, peer := range peers {
					ixns = append(ixns, &api.SourceIntention{
						Name:   "ac6-client",
						Peer:   peer,
						Action: api.IntentionActionAllow,
					})
				}
				return ixns
			}(),
		},
		Exports: func() []api.ServiceConsumer {
			var consumers []api.ServiceConsumer
			for _, peer := range peers {
				consumers = append(consumers, api.ServiceConsumer{
					Peer: peer,
				})
			}
			return consumers
		}(),
	})
}

func (s *ac6FailoversSuite) test(t *testing.T, ct *commonTopo) {
	dc1 := ct.Sprawl.Topology().Clusters["dc1"]
	dc2 := ct.Sprawl.Topology().Clusters["dc2"]

	type testcase struct {
		name      string
		cluster   *topology.Cluster
		peer      *topology.Cluster
		partition string
	}
	tcs := []testcase{
		{
			name:      "dc1 default partition failovers",
			cluster:   dc1,
			peer:      dc2, // dc3 is hardcoded
			partition: "default",
		},
		{
			name:      "dc1 part1 partition failovers",
			cluster:   dc1,
			peer:      dc2, // dc3 is hardcoded
			partition: "part1",
		},
		{
			name:      "dc2 default partition failovers",
			cluster:   dc2,
			peer:      dc1, // dc3 is hardcoded
			partition: "default",
		},
		{
			name:      "dc2 part1 partition failovers",
			cluster:   dc2,
			peer:      dc1, // dc3 is hardcoded
			partition: "part1",
		},
	}
	for _, tc := range tcs {
		t.Run(tc.name, func(t *testing.T) {
			// NOTE: *not parallel* because we mutate resources that are shared
			// between test cases (disable/enable nodes)
			if !utils.IsEnterprise() && tc.partition != "default" {
				t.Skip("skipping enterprise test")
			}
			partition := tc.partition
			clu := tc.cluster
			peerClu := tc.peer

			svcs := clu.ServicesByID(s.ac6[nodeKey{clu.Datacenter, partition}].clientSID)
			require.Len(t, svcs, 1, "expected exactly one client in datacenter")

			serverSID := s.ac6[nodeKey{clu.Datacenter, partition}].serverSID
			serverSID.Normalize()

			client := svcs[0]
			require.Len(t, client.Upstreams, 1, "expected one upstream for client")

			u := client.Upstreams[0]
			ct.Assert.CatalogServiceExists(t, clu.Name, u.ID.Name, utils.CompatQueryOpts(&api.QueryOptions{
				Partition: u.ID.Partition,
			}))

			t.Cleanup(func() {
				cfg := ct.Sprawl.Config()
				for _, part := range clu.Partitions {
					EnableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, part.Name}].serverNode)
				}
				for _, part := range peerClu.Partitions {
					EnableNode(t, cfg, peerClu.Name, s.ac6[nodeKey{peerClu.Datacenter, part.Name}].serverNode)
				}
				require.NoError(t, ct.Sprawl.Relaunch(cfg))
			})

			fmt.Println("### preconditions")
			// TODO: deduce this number, instead of hard-coding
			nFailoverTargets := 4
			// in OSS, we don't have failover targets for non-default partitions
			if !utils.IsEnterprise() {
				nFailoverTargets = 3
			}
			for i := 0; i < nFailoverTargets; i++ {
				ct.Assert.UpstreamEndpointStatus(t, client, fmt.Sprintf("failover-target~%d~%s", i, clusterPrefix(u, clu.Datacenter)), "HEALTHY", 1)
			}

			ct.Assert.FortioFetch2FortioName(t, client, u, clu.Name, serverSID)

			if t.Failed() {
				t.Fatalf("failed preconditions")
			}

			fmt.Println("### Failover to peer target")
			cfg := ct.Sprawl.Config()
			DisableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, partition}].serverNode)
			require.NoError(t, ct.Sprawl.Relaunch(cfg))
			// Clusters for imported services rely on outlier detection for
			// failovers, NOT eds_health_status. This means that killing the
			// node above does not actually make the envoy cluster UNHEALTHY
			// so we do not assert for it.
			expectUID := topology.ServiceID{
				Name:      u.ID.Name,
				Partition: "default",
			}
			expectUID.Normalize()
			ct.Assert.FortioFetch2FortioName(t, client, u, peerClu.Name, expectUID)

			if utils.IsEnterprise() {
				fmt.Println("### Failover to peer target in non-default partition")
				cfg = ct.Sprawl.Config()
				DisableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, partition}].serverNode)
				DisableNode(t, cfg, peerClu.Name, s.ac6[nodeKey{peerClu.Datacenter, "default"}].serverNode)
				require.NoError(t, ct.Sprawl.Relaunch(cfg))
				// Retry until outlier_detection deems the cluster
				// unhealthy and fails over to peer part1.
				expectUID = topology.ServiceID{
					Name:      u.ID.Name,
					Partition: "part1",
				}
				expectUID.Normalize()
				ct.Assert.FortioFetch2FortioName(t, client, u, peerClu.Name, expectUID)
			}

			fmt.Println("### Failover to dc3 peer target")
			cfg = ct.Sprawl.Config()
			DisableNode(t, cfg, clu.Name, s.ac6[nodeKey{clu.Datacenter, partition}].serverNode)
			// Disable all partitions for peer
			for _, part := range peerClu.Partitions {
				DisableNode(t, cfg, peerClu.Name, s.ac6[nodeKey{peerClu.Datacenter, part.Name}].serverNode)
			}
			require.NoError(t, ct.Sprawl.Relaunch(cfg))
			// This will retry until outlier_detection deems the cluster
			// unhealthy and fails over to dc3.
			expectUID = topology.ServiceID{
				Name:      u.ID.Name,
				Partition: "default",
			}
			expectUID.Normalize()
			ct.Assert.FortioFetch2FortioName(t, client, u, "dc3", expectUID)
		})
	}
}

func clusterPrefix(u *topology.Upstream, dc string) string {
	u.ID.Normalize()
	switch u.ID.Partition {
	case "default":
		return fmt.Sprintf("%s.%s.%s.internal", u.ID.Name, u.ID.Namespace, dc)
	default:
		return fmt.Sprintf("%s.%s.%s.%s.internal-v1", u.ID.Name, u.ID.Namespace, u.ID.Partition, dc)
	}
}