Run integration tests locally using amd64 (#14365)

Locally, always run integration tests using amd64, even if running
on an arm mac. This ensures the architecture locally always matches
the CI/CD environment.

In addition:
* Use consul:local for envoy integration and upgrade tests. Previously,
  consul:local was used for upgrade tests and consul-dev for integration
  tests. I didn't see a reason to use separate images as it's more
  confusing.
* By default, disable the requirement that aws credentials are set.
  These are only needed for the lambda tests and make it so you
  can't run any tests locally, even if you're not running the
  lambda tests. Now they'll only run if the LAMBDA_TESTS_ENABLED
  env var is set.
* Split out the building of the Docker image for integration
  tests into its own target from `dev-docker`. This allows us to always
  use an amd64 image without messing up the `dev-docker` target.
* Add support for passing GO_TEST_FLAGs to `test-envoy-integ` target.
* Add a wait_for_leader function because tests were failing locally
  without it.

.circleci/config.yml

@@ -816,7 +816,7 @@ jobs:
       # Get go binary from workspace
       - attach_workspace:
           at: .
-      # Build the consul-dev image from the already built binary
+      # Build the consul:local image from the already built binary
       - run:
           command: |
             sudo rm -rf /usr/local/go
@@ -887,8 +887,8 @@ jobs:
       - attach_workspace:
           at: .
       - run: *install-gotestsum
-      # Build the consul-dev image from the already built binary
-      - run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile .
+      # Build the consul:local image from the already built binary
+      - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
       - run:
           name: Envoy Integration Tests
           command: |
@@ -902,6 +902,7 @@ jobs:
             GOTESTSUM_JUNITFILE: /tmp/test-results/results.xml
             GOTESTSUM_FORMAT: standard-verbose
             COMPOSE_INTERACTIVE_NO_CLI: 1
+            LAMBDA_TESTS_ENABLED: "true"
             # tput complains if this isn't set to something.
             TERM: ansi
       - store_artifacts:

GNUmakefile

@@ -130,7 +130,7 @@ export GOLDFLAGS
 
 # Allow skipping docker build during integration tests in CI since we already
 # have a built binary
-ENVOY_INTEG_DEPS?=dev-docker
+ENVOY_INTEG_DEPS?=docker-envoy-integ
 ifdef SKIP_DOCKER_BUILD
 ENVOY_INTEG_DEPS=noop
 endif
@@ -346,8 +346,22 @@ consul-docker: go-build-image
 ui-docker: ui-build-image
 	@$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh ui
 
+# Build image used to run integration tests locally.
+docker-envoy-integ:
+	$(MAKE) GOARCH=amd64 linux
+	docker build \
+      --platform linux/amd64 $(NOCACHE) $(QUIET) \
+      -t 'consul:local' \
+      --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+      $(CURDIR)/pkg/bin/linux_amd64 \
+      -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile
+
+# Run integration tests.
+# Use GO_TEST_FLAGS to run specific tests:
+#      make test-envoy-integ GO_TEST_FLAGS="-run TestEnvoy/case-basic"
+# NOTE: Always uses amd64 images, even when running on M1 macs, to match CI/CD environment.
 test-envoy-integ: $(ENVOY_INTEG_DEPS)
-	@go test -v -timeout=30m -tags integration ./test/integration/connect/envoy
+	@go test -v -timeout=30m -tags integration $(GO_TEST_FLAGS) ./test/integration/connect/envoy
 
 .PHONY: test-compat-integ
 test-compat-integ: dev-docker

test/integration/connect/envoy/Dockerfile-consul-envoy

@@ -1,7 +1,7 @@
 # Note this arg has to be before the first FROM
 ARG ENVOY_VERSION
 
-FROM consul-dev as consul
+FROM consul:local as consul
 
 FROM docker.mirror.hashicorp.services/envoyproxy/envoy:v${ENVOY_VERSION}
 COPY --from=consul /bin/consul /bin/consul

test/integration/connect/envoy/case-wanfed-gw/global-setup.sh

@@ -17,7 +17,7 @@ consul tls cert create -dc=secondary -server -node=sec
 "
 
 docker rm -f "$container" &>/dev/null || true
-docker run -i --net=none --name="$container" consul-dev:latest sh -c "${scriptlet}"
+docker run -i --net=none --name="$container" consul:local sh -c "${scriptlet}"
 
 # primary
 for f in \

test/integration/connect/envoy/helpers.bash

@@ -562,14 +562,14 @@ function assert_intention_denied {
 function docker_consul {
   local DC=$1
   shift 1
-  docker run -i --rm --network container:envoy_consul-${DC}_1 consul-dev "$@"
+  docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@"
 }
 
 function docker_consul_for_proxy_bootstrap {
   local DC=$1
   shift 1
 
-  docker run -i --rm --network container:envoy_consul-${DC}_1 consul-dev "$@"
+  docker run -i --rm --network container:envoy_consul-${DC}_1 consul:local "$@" 2> /dev/null
 }
 
 function docker_wget {
@@ -581,7 +581,7 @@ function docker_wget {
 function docker_curl {
   local DC=$1
   shift 1
-  docker run --rm --network container:envoy_consul-${DC}_1 --entrypoint curl consul-dev "$@"
+  docker run --rm --network container:envoy_consul-${DC}_1 --entrypoint curl consul:local "$@"
 }
 
 function docker_exec {
@@ -806,9 +806,16 @@ function delete_config_entry {
 
 function register_services {
   local DC=${1:-primary}
+  wait_for_leader "$DC"
   docker_consul_exec ${DC} sh -c "consul services register /workdir/${DC}/register/service_*.hcl"
 }
 
+# wait_for_leader waits until a leader is elected.
+# Its first argument must be the datacenter name.
+function wait_for_leader {
+  retry_default docker_consul_exec "$1" sh -c '[[ $(curl --fail -sS http://127.0.0.1:8500/v1/status/leader) ]]'
+}
+
 function setup_upsert_l4_intention {
   local SOURCE=$1
   local DESTINATION=$2

test/integration/connect/envoy/run-tests.sh

@@ -16,6 +16,8 @@ ENVOY_VERSION=${ENVOY_VERSION:-"1.23.0"}
 export ENVOY_VERSION
 
 export DOCKER_BUILDKIT=1
+# Always run tests on amd64 because that's what the CI environment uses.
+export DOCKER_DEFAULT_PLATFORM="linux/amd64"
 
 if [ ! -z "$DEBUG" ] ; then
   set -x
@@ -44,17 +46,19 @@ function network_snippet {
 }
 
 function aws_snippet {
-  local snippet=""
-
-  # The Lambda integration cases assume that a Lambda function exists in $AWS_REGION with an ARN of $AWS_LAMBDA_ARN.
-  # The AWS credentials must have permission to invoke the Lambda function.
-  [ -n "$(set | grep '^AWS_ACCESS_KEY_ID=')" ] && snippet="${snippet} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID"
-  [ -n "$(set | grep '^AWS_SECRET_ACCESS_KEY=')" ] && snippet="${snippet} -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY"
-  [ -n "$(set | grep '^AWS_SESSION_TOKEN=')" ] && snippet="${snippet} -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN"
-  [ -n "$(set | grep '^AWS_LAMBDA_REGION=')" ] && snippet="${snippet} -e AWS_LAMBDA_REGION=$AWS_LAMBDA_REGION"
-  [ -n "$(set | grep '^AWS_LAMBDA_ARN=')" ] && snippet="${snippet} -e AWS_LAMBDA_ARN=$AWS_LAMBDA_ARN"
-
-  echo "$snippet"
+  if [[ ! -z "$LAMBDA_TESTS_ENABLED" ]]; then
+    local snippet=""
+
+    # The Lambda integration cases assume that a Lambda function exists in $AWS_REGION with an ARN of $AWS_LAMBDA_ARN.
+    # The AWS credentials must have permission to invoke the Lambda function.
+    [ -n "$(set | grep '^AWS_ACCESS_KEY_ID=')" ] && snippet="${snippet} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID"
+    [ -n "$(set | grep '^AWS_SECRET_ACCESS_KEY=')" ] && snippet="${snippet} -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY"
+    [ -n "$(set | grep '^AWS_SESSION_TOKEN=')" ] && snippet="${snippet} -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN"
+    [ -n "$(set | grep '^AWS_LAMBDA_REGION=')" ] && snippet="${snippet} -e AWS_LAMBDA_REGION=$AWS_LAMBDA_REGION"
+    [ -n "$(set | grep '^AWS_LAMBDA_ARN=')" ] && snippet="${snippet} -e AWS_LAMBDA_ARN=$AWS_LAMBDA_ARN"
+
+    echo "$snippet"
+  fi
 }
 
 function init_workdir {
@@ -222,7 +226,7 @@ function start_consul {
       --hostname "consul-${DC}-server" \
       --network-alias "consul-${DC}-server" \
       -e "CONSUL_LICENSE=$license" \
-      consul-dev \
+      consul:local \
       agent -dev -datacenter "${DC}" \
       -config-dir "/workdir/${DC}/consul" \
       -config-dir "/workdir/${DC}/consul-server" \
@@ -237,7 +241,7 @@ function start_consul {
       --network-alias "consul-${DC}-client" \
       -e "CONSUL_LICENSE=$license" \
       ${ports[@]} \
-      consul-dev \
+      consul:local \
       agent -datacenter "${DC}" \
       -config-dir "/workdir/${DC}/consul" \
       -data-dir "/tmp/consul" \
@@ -256,7 +260,7 @@ function start_consul {
       --network-alias "consul-${DC}-server" \
       -e "CONSUL_LICENSE=$license" \
       ${ports[@]} \
-      consul-dev \
+      consul:local \
       agent -dev -datacenter "${DC}" \
       -config-dir "/workdir/${DC}/consul" \
       -config-dir "/workdir/${DC}/consul-server" \
@@ -290,7 +294,7 @@ function start_partitioned_client {
     --hostname "consul-${PARTITION}-client" \
     --network-alias "consul-${PARTITION}-client" \
     -e "CONSUL_LICENSE=$license" \
-    consul-dev agent \
+    consul:local agent \
     -datacenter "primary" \
     -retry-join "consul-primary-server" \
     -grpc-port 8502 \