diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index 24c5b37bd5..3c4eeafc13 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -22,6 +22,7 @@ type Agent interface { GetConfig() Config GetInfo() AgentInfo GetDatacenter() string + GetNetwork() string IsServer() bool RegisterTermination(func() error) Terminate() error diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index c569a21a38..ba24b798c7 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -63,7 +63,7 @@ func NewN(t TestingT, conf Config, count int) (*Cluster, error) { // // The provided TestingT is used to register a cleanup function to terminate // the cluster. -func New(t TestingT, configs []Config) (*Cluster, error) { +func New(t TestingT, configs []Config, ports ...int) (*Cluster, error) { id, err := shortid.Generate() if err != nil { return nil, fmt.Errorf("could not cluster id: %w", err) @@ -99,7 +99,7 @@ func New(t TestingT, configs []Config) (*Cluster, error) { _ = cluster.Terminate() }) - if err := cluster.Add(configs, true); err != nil { + if err := cluster.Add(configs, true, ports...); err != nil { return nil, fmt.Errorf("could not start or join all agents: %w", err) } @@ -115,7 +115,7 @@ func (c *Cluster) AddN(conf Config, count int, join bool) error { } // Add starts an agent with the given configuration and joins it with the existing cluster -func (c *Cluster) Add(configs []Config, serfJoin bool) (xe error) { +func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) { if c.Index == 0 && !serfJoin { return fmt.Errorf("the first call to Cluster.Add must have serfJoin=true") } @@ -135,6 +135,7 @@ func (c *Cluster) Add(configs []Config, serfJoin bool) (xe error) { context.Background(), conf, c, + ports..., ) if err != nil { return fmt.Errorf("could not add container index %d: %w", idx, err) diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index bd4416a35b..50df1b072f 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -73,7 +73,7 @@ func (c *consulContainerNode) ClaimAdminPort() (int, error) { } // NewConsulContainer starts a Consul agent in a container with the given config. -func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (Agent, error) { +func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, ports ...int) (Agent, error) { network := cluster.NetworkName index := cluster.Index if config.ScratchDir == "" { @@ -128,7 +128,7 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (A addtionalNetworks: []string{"bridge", network}, hostname: fmt.Sprintf("agent-%d", index), } - podReq, consulReq := newContainerRequest(config, opts) + podReq, consulReq := newContainerRequest(config, opts, ports...) // Do some trickery to ensure that partial completion is correctly torn // down, but successful execution is not. @@ -291,6 +291,10 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (A return node, nil } +func (c *consulContainerNode) GetNetwork() string { + return c.network +} + func (c *consulContainerNode) GetName() string { if c.container == nil { return c.consulReq.Name // TODO: is this safe to do all the time? @@ -501,7 +505,7 @@ type containerOpts struct { addtionalNetworks []string } -func newContainerRequest(config Config, opts containerOpts) (podRequest, consulRequest testcontainers.ContainerRequest) { +func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRequest, consulRequest testcontainers.ContainerRequest) { skipReaper := isRYUKDisabled() pod := testcontainers.ContainerRequest{ @@ -539,6 +543,10 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", basePort+i)) } + for _, port := range ports { + pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", port)) + } + // For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready". // If we only wait until the log stream starts, subsequent commands to agents will fail. // TODO: optimize the wait strategy diff --git a/test/integration/consul-container/libs/service/connect.go b/test/integration/consul-container/libs/service/connect.go index 49a340bd2a..c7c064aa4e 100644 --- a/test/integration/consul-container/libs/service/connect.go +++ b/test/integration/consul-container/libs/service/connect.go @@ -58,6 +58,10 @@ func (g ConnectContainer) GetAddrs() (string, []int) { return g.ip, g.appPort } +func (g ConnectContainer) GetPort(port int) int { + return 0 +} + func (g ConnectContainer) Restart() error { _, err := g.GetStatus() if err != nil { diff --git a/test/integration/consul-container/libs/service/examples.go b/test/integration/consul-container/libs/service/examples.go index 3d6258eaa9..1955b01f69 100644 --- a/test/integration/consul-container/libs/service/examples.go +++ b/test/integration/consul-container/libs/service/examples.go @@ -68,6 +68,10 @@ func (g exampleContainer) GetAddrs() (string, []int) { return "", nil } +func (g exampleContainer) GetPort(port int) int { + return 0 +} + func (g exampleContainer) Restart() error { return fmt.Errorf("Restart Unimplemented by ConnectContainer") } diff --git a/test/integration/consul-container/libs/service/gateway.go b/test/integration/consul-container/libs/service/gateway.go index 7028a61292..646dc52a9c 100644 --- a/test/integration/consul-container/libs/service/gateway.go +++ b/test/integration/consul-container/libs/service/gateway.go @@ -20,12 +20,13 @@ import ( // gatewayContainer type gatewayContainer struct { - ctx context.Context - container testcontainers.Container - ip string - port int - adminPort int - serviceName string + ctx context.Context + container testcontainers.Container + ip string + port int + adminPort int + serviceName string + portMappings map[int]int } var _ Service = (*gatewayContainer)(nil) @@ -105,6 +106,10 @@ func (g gatewayContainer) GetAdminAddr() (string, int) { return "localhost", g.adminPort } +func (g gatewayContainer) GetPort(port int) int { + return g.portMappings[port] +} + func (g gatewayContainer) Restart() error { _, err := g.container.State(g.ctx) if err != nil { @@ -130,7 +135,7 @@ func (g gatewayContainer) GetStatus() (string, error) { return state.Status, err } -func NewGatewayService(ctx context.Context, name string, kind string, node libcluster.Agent) (Service, error) { +func NewGatewayService(ctx context.Context, name string, kind string, node libcluster.Agent, ports ...int) (Service, error) { nodeConfig := node.GetConfig() if nodeConfig.ScratchDir == "" { return nil, fmt.Errorf("node ScratchDir is required") @@ -207,21 +212,33 @@ func NewGatewayService(ctx context.Context, name string, kind string, node libcl adminPortStr = strconv.Itoa(adminPort) ) - info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{ + extraPorts := []string{} + for _, port := range ports { + extraPorts = append(extraPorts, strconv.Itoa(port)) + } + + info, err := cluster.LaunchContainerOnNode(ctx, node, req, append( + extraPorts, portStr, adminPortStr, - }) + )) if err != nil { return nil, err } + portMappings := make(map[int]int) + for _, port := range ports { + portMappings[port] = info.MappedPorts[strconv.Itoa(port)].Int() + } + out := &gatewayContainer{ - ctx: ctx, - container: info.Container, - ip: info.IP, - port: info.MappedPorts[portStr].Int(), - adminPort: info.MappedPorts[adminPortStr].Int(), - serviceName: name, + ctx: ctx, + container: info.Container, + ip: info.IP, + port: info.MappedPorts[portStr].Int(), + adminPort: info.MappedPorts[adminPortStr].Int(), + serviceName: name, + portMappings: portMappings, } return out, nil diff --git a/test/integration/consul-container/libs/service/service.go b/test/integration/consul-container/libs/service/service.go index f32bd67ff8..ac8a9a70a0 100644 --- a/test/integration/consul-container/libs/service/service.go +++ b/test/integration/consul-container/libs/service/service.go @@ -2,6 +2,7 @@ package service import ( "context" + "github.com/hashicorp/consul/api" ) @@ -13,6 +14,7 @@ type Service interface { Export(partition, peer string, client *api.Client) error GetAddr() (string, int) GetAddrs() (string, []int) + GetPort(port int) int // GetAdminAddr returns the external admin address GetAdminAddr() (string, int) GetLogs() (string, error) diff --git a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go index 07bd600e69..8e8e4b6f91 100644 --- a/test/integration/consul-container/test/gateways/gateway_endpoint_test.go +++ b/test/integration/consul-container/test/gateways/gateway_endpoint_test.go @@ -1,10 +1,7 @@ package gateways_test import ( - "bufio" - "bytes" "context" - "fmt" "github.com/hashicorp/consul/api" libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" @@ -12,11 +9,7 @@ import ( "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "net/http" - "os" - "strconv" "testing" - "time" ) func TestAPIGatewayCreate(t *testing.T) { @@ -26,7 +19,9 @@ func TestAPIGatewayCreate(t *testing.T) { t.Parallel() - cluster := createCluster(t) + listenerPortOne := 6000 + + cluster := createCluster(t, listenerPortOne) client := cluster.APIClient(0) @@ -36,7 +31,7 @@ func TestAPIGatewayCreate(t *testing.T) { Name: "api-gateway", Listeners: []api.APIGatewayListener{ { - Port: 8443, + Port: listenerPortOne, Protocol: "tcp", }, }, @@ -64,8 +59,7 @@ func TestAPIGatewayCreate(t *testing.T) { assert.NoError(t, err) // Create a client proxy instance with the server as an upstream - clientConnectProxy, gatewayService := createServices(t, cluster) - fmt.Println(clientConnectProxy) + _, gatewayService := createServices(t, cluster, listenerPortOne) //how to exec into the consul CLI //agentUrl, err := cluster.Agents[0].GetPod().PortEndpoint(context.Background(), "8500", "http") @@ -93,65 +87,16 @@ func TestAPIGatewayCreate(t *testing.T) { //check status entry, _, err := client.ConfigEntries().Get("api-gateway", "api-gateway", nil) assert.NoError(t, err) - t.Log(entry) apiEntry := entry.(*api.APIGatewayConfigEntry) - t.Log(apiEntry.Status) gatewayReady = isAccepted(apiEntry.Status.Conditions) e, _, err := client.ConfigEntries().Get("tcp-route", "api-gateway-route", nil) assert.NoError(t, err) - t.Log(entry) routeEntry := e.(*api.TCPRouteConfigEntry) - t.Log(routeEntry.Status) routeReady = isBound(routeEntry.Status.Conditions) - //this doesn't seem to actually do anything - time.Sleep(10 * time.Second) } - agentServices, err := client.Agent().Services() - assert.NoError(t, err) - for _, s := range agentServices { - t.Log(s) - } - - services, _, err := client.Catalog().Services(nil) - assert.NoError(t, err) - for key, s := range services { - t.Log(key, s) - } - - gateways, _, err := client.Catalog().GatewayServices("api-gateway", nil) - assert.NoError(t, err) - for _, g := range gateways { - t.Log(g) - } - - t.Log(gatewayService.GetAddr()) - assert.NoError(t, err) - fmt.Println(gatewayService) - - ip, port := gatewayService.GetAddr() - t.Log("ip:", ip) - stdOut := bufio.NewWriter(os.Stdout) - stdOut.Write([]byte(ip + "\n")) - stdOut.Write([]byte(strconv.Itoa(port))) - stdOut.Flush() - resp, err := http.Get(fmt.Sprintf("http://%s:%d", ip, port)) - t.Log(resp, err) - assert.NoError(t, err) - - buf := bytes.NewBufferString("abcdefg") - resp, err = http.Post(fmt.Sprintf("http://%s:%d", ip, port), "text/plain", buf) - t.Log(resp, err) - - for { - } - //t.Log(gatewayService.Restart()) - //t.Log(gatewayService.GetStatus()) - //t.Log(gatewayService.GetLogs()) - - t.Fail() - + libassert.HTTPServiceEchoes(t, "localhost", gatewayService.GetPort(listenerPortOne), "") } func isAccepted(conditions []api.Condition) bool { @@ -172,7 +117,7 @@ func conditionStatusIsValue(typeName string, statusValue string, conditions []ap } // TODO this code is just copy pasted from elsewhere, it is likely we will need to modify it some -func createCluster(t *testing.T) *libcluster.Cluster { +func createCluster(t *testing.T, ports ...int) *libcluster.Cluster { opts := libcluster.BuildOptions{ InjectAutoEncryption: true, InjectGossipEncryption: true, @@ -186,7 +131,7 @@ func createCluster(t *testing.T) *libcluster.Cluster { configs := []libcluster.Config{*conf} - cluster, err := libcluster.New(t, configs) + cluster, err := libcluster.New(t, configs, ports...) require.NoError(t, err) node := cluster.Agents[0] @@ -205,7 +150,7 @@ func createCluster(t *testing.T) *libcluster.Cluster { return cluster } -func createServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Service, libservice.Service) { +func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (libservice.Service, libservice.Service) { node := cluster.Agents[0] client := node.GetClient() // Create a service and proxy instance @@ -226,12 +171,11 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Servi // Create a client proxy instance with the server as an upstream clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) require.NoError(t, err) - libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy") - gatewayService, err := libservice.NewGatewayService(context.Background(), "api-gateway", "api", cluster.Agents[0]) - libassert.CatalogServiceExists(t, client, "api-gateway") + gatewayService, err := libservice.NewGatewayService(context.Background(), "api-gateway", "api", cluster.Agents[0], ports...) require.NoError(t, err) + libassert.CatalogServiceExists(t, client, "api-gateway") return clientConnectProxy, gatewayService }