consul/test-integ/peering_commontopo/ac6_failovers_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package peering

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
	"github.com/hashicorp/consul/testing/deployer/sprawl"
	"github.com/hashicorp/consul/testing/deployer/topology"
)

type ac6FailoversSuite struct {
	// inputs
	// with all false, this gives us a scenario with:
	// - a "near" server in the accepter cluster (DC1), partitition default, namespace default
	// - a "far" server in the dialer cluster (DC2), partition default, namespace default
	// - a client in the accepter cluster (DC1), partition default, namespace default, with:
	//   - upstream near server (DC1)
	//   - failover to far server (DC2)
	//
	// TODO: technically if NearInDial && !FarInAcc (i.e., near == far), then we're not doing peering at all,
	// and could do this test in a single DC

	// when true, put the client (and its default upstream server) in the dialer peer; otherwise, put client in accepter
	NearInDial bool
	// when true, put the client (and its default upstream server) in the nondefault partition/namespace; otherwise in the default
	NearInPartAlt bool
	NearInNSAlt   bool
	// when true, put far server to the accepter peer; otherwise the dialer
	FarInAcc bool
	// when true, put far server to nondefault partition/namespace (ENT-only); otherwise, failover to default
	FarInPartAlt bool
	FarInNSAlt   bool

	// launch outputs, for querying during test
	clientSID topology.ID
	// near = same DC as client; far = other DC
	nearServerSID topology.ID
	// used to remove the node and trigger failover
	nearServerNode topology.NodeID
	farServerSID   topology.ID
	farServerNode  topology.NodeID
}

// Note: this test cannot share topo
func TestAC6Failovers(t *testing.T) {
	// bit banging to get all permutations of all params
	const nParams = 3
	// i.e 2**nParams
	const n = int(1) << nParams
	for i := 0; i < n; i++ {
		s := ac6FailoversSuite{
			// xth bit == 1
			NearInDial:    (i>>0)&1 == 1,
			NearInPartAlt: (i>>1)&1 == 1,
			FarInPartAlt:  (i>>2)&1 == 1,
		}
		// ensure the servers are always in separate DCs
		s.FarInAcc = s.NearInDial
		t.Run(fmt.Sprintf("%02d_%s", i, s.testName()), func(t *testing.T) {
			t.Parallel()
			ct := NewCommonTopo(t)
			s.setup(t, ct)
			ct.Launch(t)
			s.test(t, ct)
		})
	}
}

func TestNET5029Failovers(t *testing.T) {
	// TODO: *.{a,b} are not actually peering tests, and should technically be moved elsewhere
	suites := map[string]ac6FailoversSuite{
		"1.a": {
			FarInAcc:     true,
			FarInPartAlt: true,
		},
		"1.b": {
			FarInAcc:   true,
			FarInNSAlt: true,
		},
		"1.c": {
			FarInNSAlt: true,
		},
		"1.d": {
			FarInPartAlt: true,
		},
		"2.a": {
			FarInAcc:      true,
			NearInPartAlt: true,
		},
		"2.b": {
			FarInAcc:    true,
			NearInNSAlt: true,
		},
		"2.c": {
			NearInDial:  true,
			NearInNSAlt: true,
			FarInAcc:    true,
		},
		"2.d": {
			NearInDial:    true,
			NearInPartAlt: true,
			FarInAcc:      true,
		},
	}
	for name, s := range suites {
		s := s
		t.Run(fmt.Sprintf("%s_%s", name, s.testName()), func(t *testing.T) {
			if name == "1.b" {
				t.Skip("TODO: fails with 503/504")
			}
			t.Parallel()
			ct := NewCommonTopo(t)
			s.setup(t, ct)
			ct.Launch(t)
			s.test(t, ct)
		})
	}
}

func TestAC6Failovers_AllPermutations(t *testing.T) {
	//
	t.Skip("Too many permutations")
	// bit banging to get all permutations of all params
	const nParams = 6
	// i.e 2**nParams
	const n = int(1) << nParams
	for i := 0; i < n; i++ {
		s := ac6FailoversSuite{
			// xth bit == 1
			NearInDial:    (i>>0)&1 == 1,
			FarInAcc:      (i>>1)&1 == 1,
			NearInPartAlt: (i>>2)&1 == 1,
			FarInPartAlt:  (i>>3)&1 == 1,
			NearInNSAlt:   (i>>4)&1 == 1,
			FarInNSAlt:    (i>>5)&1 == 1,
		}
		t.Run(fmt.Sprintf("%02d_%s", i, s.testName()), func(t *testing.T) {
			t.Parallel()
			ct := NewCommonTopo(t)
			s.setup(t, ct)
			ct.Launch(t)
			s.test(t, ct)
		})
	}
}

func (s *ac6FailoversSuite) testName() (ret string) {
	switch s.NearInDial {
	case true:
		ret += "dial"
	default:
		ret += "acc"
	}
	ret += "."
	switch s.NearInPartAlt {
	case true:
		ret += "alt"
	default:
		ret += "default"
	}
	ret += "."
	switch s.NearInNSAlt {
	case true:
		ret += "alt"
	default:
		ret += "default"
	}

	ret += "->"

	switch s.FarInAcc {
	case true:
		ret += "acc"
	default:
		ret += "dial"
	}
	ret += "."
	switch s.FarInPartAlt {
	case true:
		ret += "alt"
	default:
		ret += "default"
	}
	ret += "."
	switch s.FarInNSAlt {
	case true:
		ret += "alt"
	default:
		ret += "default"
	}

	return
}

func (s *ac6FailoversSuite) setup(t *testing.T, ct *commonTopo) {
	if !utils.IsEnterprise() {
		if s.NearInPartAlt || s.FarInPartAlt {
			t.Skip("ENT required for nondefault partitions")
		}
		if s.NearInNSAlt || s.FarInNSAlt {
			t.Skip("ENT required for nondefault namespaces")
		}
	}

	nearClu := ct.DC1
	farClu := ct.DC2
	if s.NearInDial {
		nearClu = ct.DC2
	}
	if s.FarInAcc {
		farClu = ct.DC1
	}

	// - server in clientPartition/DC (main target)
	nearServerSID := topology.ID{
		Name:      "ac6-server",
		Partition: defaultToEmptyForCE("default"),
		Namespace: defaultToEmptyForCE("default"),
	}
	if s.NearInPartAlt {
		nearServerSID.Partition = "part1"
	}
	if s.NearInNSAlt {
		nearServerSID.Namespace = "ns1"
	}
	nearServer := NewFortioServiceWithDefaults(
		nearClu.Datacenter,
		nearServerSID,
		nil,
	)
	nearServerNode := ct.AddServiceNode(nearClu, serviceExt{Workload: nearServer})

	nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
		&api.ServiceConfigEntry{
			Kind:      api.ServiceDefaults,
			Name:      nearServerSID.Name,
			Partition: defaultToEmptyForCE(nearServerSID.Partition),
			Namespace: defaultToEmptyForCE(nearServerSID.Namespace),
			Protocol:  "http",
		},
	)
	// - server in otherPartition/otherDC
	farServerSID := topology.ID{
		Name:      nearServerSID.Name,
		Partition: defaultToEmptyForCE("default"),
		Namespace: defaultToEmptyForCE("default"),
	}
	if s.FarInPartAlt {
		farServerSID.Partition = "part1"
	}
	if s.FarInNSAlt {
		farServerSID.Namespace = "ns1"
	}
	farServer := NewFortioServiceWithDefaults(
		farClu.Datacenter,
		farServerSID,
		nil,
	)
	farServerNode := ct.AddServiceNode(farClu, serviceExt{Workload: farServer})
	if nearClu != farClu {
		ct.ExportService(farClu, farServerSID.Partition,
			api.ExportedService{
				Name:      farServerSID.Name,
				Namespace: defaultToEmptyForCE(farServerSID.Namespace),
				Consumers: []api.ServiceConsumer{
					{
						Peer: LocalPeerName(nearClu, nearServerSID.Partition),
					},
				},
			},
		)
	} else if nearClu == farClu && farServerSID.Partition != nearServerSID.Partition {
		ct.ExportService(farClu, farServerSID.Partition,
			api.ExportedService{
				Name:      farServerSID.Name,
				Namespace: defaultToEmptyForCE(farServerSID.Namespace),
				Consumers: []api.ServiceConsumer{
					{
						// this must not be "", or else it is basically ignored altogether
						// TODO: bug? if this whole struct is empty, that should be an error
						Partition: topology.PartitionOrDefault(nearServerSID.Partition),
					},
				},
			},
		)
	}

	var targets []api.ServiceResolverFailoverTarget
	if nearClu != farClu {
		targets = []api.ServiceResolverFailoverTarget{
			{
				Service:   farServerSID.Name,
				Peer:      LocalPeerName(farClu, farServerSID.Partition),
				Namespace: defaultToEmptyForCE(farServerSID.Namespace),
			},
		}
	} else {
		part := ConfigEntryPartition(farServerSID.Partition)
		// weird exception here where target partition set to "" means "inherit from parent"
		// TODO: bug? docs say "" -> default:
		// https://developer.hashicorp.com/consul/docs/connect/config-entries/service-resolver#failover-targets-partition
		if farServerSID.Partition == "default" && nearServerSID.Partition != "default" {
			part = "default"
		}
		targets = []api.ServiceResolverFailoverTarget{
			{
				Service:   farServerSID.Name,
				Partition: defaultToEmptyForCE(part),
				Namespace: defaultToEmptyForCE(farServerSID.Namespace),
			},
		}
	}

	nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
		&api.ServiceConfigEntry{
			Kind:      api.ServiceDefaults,
			Name:      farServerSID.Name,
			Partition: defaultToEmptyForCE(farServerSID.Partition),
			Namespace: defaultToEmptyForCE(farServerSID.Namespace),
			Protocol:  "http",
		},
		&api.ServiceResolverConfigEntry{
			Kind:      api.ServiceResolver,
			Name:      nearServerSID.Name,
			Partition: defaultToEmptyForCE(nearServerSID.Partition),
			Namespace: defaultToEmptyForCE(nearServerSID.Namespace),
			Failover: map[string]api.ServiceResolverFailover{
				"*": {
					Targets: targets,
				},
			},
		},
	)

	clientSID := topology.ID{
		Name:      "ac6-client",
		Partition: defaultToEmptyForCE(nearServerSID.Partition),
		Namespace: defaultToEmptyForCE(nearServerSID.Namespace),
	}
	client := NewFortioServiceWithDefaults(
		nearClu.Datacenter,
		clientSID,
		func(s *topology.Workload) {
			// Upstream per partition
			s.Upstreams = []*topology.Upstream{
				{
					ID: topology.ID{
						Name:      nearServerSID.Name,
						Partition: defaultToEmptyForCE(nearServerSID.Partition),
						Namespace: defaultToEmptyForCE(nearServerSID.Namespace),
					},
					LocalPort: 5000,
					// exposed so we can hit it directly
					// TODO: we shouldn't do this; it's not realistic
					LocalAddress: "0.0.0.0",
				},
			}
		},
	)
	ct.AddServiceNode(nearClu, serviceExt{Workload: client})
	nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
		&api.ServiceConfigEntry{
			Kind:      api.ServiceDefaults,
			Name:      clientSID.Name,
			Partition: defaultToEmptyForCE(clientSID.Partition),
			Namespace: defaultToEmptyForCE(clientSID.Namespace),
			Protocol:  "http",
		},
	)

	// intentions
	nearClu.InitialConfigEntries = append(nearClu.InitialConfigEntries,
		&api.ServiceIntentionsConfigEntry{
			Kind:      api.ServiceIntentions,
			Name:      nearServerSID.Name,
			Partition: defaultToEmptyForCE(nearServerSID.Partition),
			Namespace: defaultToEmptyForCE(nearServerSID.Namespace),
			Sources: []*api.SourceIntention{{
				Name:      clientSID.Name,
				Namespace: defaultToEmptyForCE(clientSID.Namespace),
				// in this field, "" -> destination partition, so no ConfigEntryPartition :eyeroll:
				// https://developer.hashicorp.com/consul/docs/connect/config-entries/service-intentions#sources-partition
				Partition: defaultToEmptyForCE(clientSID.Partition),
				Action:    api.IntentionActionAllow,
			}},
		},
	)
	farSource := api.SourceIntention{
		Name:      clientSID.Name,
		Namespace: defaultToEmptyForCE(clientSID.Namespace),
		Peer:      LocalPeerName(nearClu, clientSID.Partition),
		Action:    api.IntentionActionAllow,
	}
	if nearClu == farClu {
		farSource.Peer = ""
		// in this field, "" -> destination partition, so no ConfigEntryPartition :eyeroll:
		// https://developer.hashicorp.com/consul/docs/connect/config-entries/service-intentions#sources-partition
		farSource.Partition = topology.PartitionOrDefault(clientSID.Partition)
	}
	farClu.InitialConfigEntries = append(farClu.InitialConfigEntries,
		&api.ServiceIntentionsConfigEntry{
			Kind:      api.ServiceIntentions,
			Name:      farServerSID.Name,
			Partition: defaultToEmptyForCE(farServerSID.Partition),
			Namespace: defaultToEmptyForCE(farServerSID.Namespace),
			Sources:   []*api.SourceIntention{&farSource},
		},
	)

	s.clientSID = clientSID
	s.nearServerSID = nearServerSID
	s.farServerSID = farServerSID
	s.nearServerNode = nearServerNode.ID()
	s.farServerNode = farServerNode.ID()
}

func (s *ac6FailoversSuite) test(t *testing.T, ct *commonTopo) {
	// NOTE: *not parallel* because we mutate resources that are shared
	// between test cases (disable/enable nodes)

	nearClu := ct.Sprawl.Topology().Clusters["dc1"]
	farClu := ct.Sprawl.Topology().Clusters["dc2"]
	if s.NearInDial {
		nearClu = ct.Sprawl.Topology().Clusters["dc2"]
	}
	if s.FarInAcc {
		farClu = ct.Sprawl.Topology().Clusters["dc1"]
	}

	svcs := nearClu.WorkloadsByID(s.clientSID)
	require.Len(t, svcs, 1, "expected exactly one client in datacenter")

	client := svcs[0]
	require.Len(t, client.Upstreams, 1, "expected one upstream for client")
	upstream := client.Upstreams[0]

	fmt.Println("### preconditions")

	// this is the server in the same DC and partitions as client
	serverSID := s.nearServerSID
	serverSID.Normalize()
	ct.Assert.FortioFetch2FortioName(t, client, upstream, nearClu.Name, serverSID)

	ct.Assert.CatalogServiceExists(t, nearClu.Name, upstream.ID.Name, utils.CompatQueryOpts(&api.QueryOptions{
		Partition: upstream.ID.Partition,
		Namespace: upstream.ID.Namespace,
	}))

	if t.Failed() {
		t.Fatal("failed preconditions")
	}

	fmt.Println("### failover")

	cfg := ct.Sprawl.Config()
	DisableNode(t, cfg, nearClu.Name, s.nearServerNode)
	require.NoError(t, ct.Sprawl.RelaunchWithPhase(cfg, sprawl.LaunchPhaseRegular))
	// Clusters for imported services rely on outlier detection for
	// failovers, NOT eds_health_status. This means that killing the
	// node above does not actually make the envoy cluster UNHEALTHY
	// so we do not assert for it.
	expectSID := s.farServerSID
	expectSID.Normalize()
	ct.Assert.FortioFetch2FortioName(t, client, upstream, farClu.Name, expectSID)
}

func defaultToEmptyForCE(tenancy string) string {
	if utils.IsEnterprise() {
		return tenancy
	}
	return topology.DefaultToEmpty(tenancy)
}