# For documentation on building consul from source, refer to: # https://www.consul.io/docs/install#compiling-from-source SHELL = bash ### # These version variables can either be a valid string for "go install @" # or the string @DEV to imply use what is currently installed locally. ### GOLANGCI_LINT_VERSION='v1.51.1' MOCKERY_VERSION='v2.20.0' BUF_VERSION='v1.14.0' PROTOC_GEN_GO_GRPC_VERSION="v1.2.0" MOG_VERSION='v0.4.0' PROTOC_GO_INJECT_TAG_VERSION='v1.3.0' PROTOC_GEN_GO_BINARY_VERSION="v0.1.0" DEEP_COPY_VERSION='bc3f5aa5735d8a54961580a3a24422c308c831c2' MOCKED_PB_DIRS= pbdns GOTAGS ?= GOPATH=$(shell go env GOPATH) GOARCH?=$(shell go env GOARCH) MAIN_GOPATH=$(shell go env GOPATH | cut -d: -f1) export PATH := $(PWD)/bin:$(GOPATH)/bin:$(PATH) # Get the git commit GIT_COMMIT?=$(shell git rev-parse --short HEAD) GIT_COMMIT_YEAR?=$(shell git show -s --format=%cd --date=format:%Y HEAD) GIT_DIRTY?=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true) GIT_IMPORT=github.com/hashicorp/consul/version DATE_FORMAT="%Y-%m-%dT%H:%M:%SZ" # it's tricky to do an RFC3339 format in a cross platform way, so we hardcode UTC GIT_DATE=$(shell $(CURDIR)/build-support/scripts/build-date.sh) # we're using this for build date because it's stable across platform builds GOLDFLAGS=-X $(GIT_IMPORT).GitCommit=$(GIT_COMMIT)$(GIT_DIRTY) -X $(GIT_IMPORT).BuildDate=$(GIT_DATE) GOTESTSUM_PATH?=$(shell command -v gotestsum) ifeq ($(FORCE_REBUILD),1) NOCACHE=--no-cache else NOCACHE= endif DOCKER_BUILD_QUIET?=1 ifeq (${DOCKER_BUILD_QUIET},1) QUIET=-q else QUIET= endif ifeq ("$(GOTAGS)","") CONSUL_COMPAT_TEST_IMAGE=consul else CONSUL_COMPAT_TEST_IMAGE=hashicorp/consul-enterprise endif CONSUL_DEV_IMAGE?=consul-dev GO_BUILD_TAG?=consul-build-go UI_BUILD_TAG?=consul-build-ui BUILD_CONTAINER_NAME?=consul-builder CONSUL_IMAGE_VERSION?=latest ENVOY_VERSION?='1.25.4' ################ # CI Variables # ################ CI_DEV_DOCKER_NAMESPACE?=hashicorpdev CI_DEV_DOCKER_IMAGE_NAME?=consul CI_DEV_DOCKER_WORKDIR?=bin/ ################ TEST_MODCACHE?=1 TEST_BUILDCACHE?=1 # You can only use as many CPUs as you have allocated to docker ifdef TEST_DOCKER_CPUS TEST_DOCKER_RESOURCE_CONSTRAINTS=--cpus $(TEST_DOCKER_CPUS) TEST_PARALLELIZATION=-e GOMAXPROCS=$(TEST_DOCKER_CPUS) else TEST_DOCKER_RESOURCE_CONSTRAINTS= TEST_PARALLELIZATION= endif ifeq ($(TEST_MODCACHE), 1) TEST_MODCACHE_VOL=-v $(MAIN_GOPATH)/pkg/mod:/go/pkg/mod else TEST_MODCACHE_VOL= endif ifeq ($(TEST_BUILDCACHE), 1) TEST_BUILDCACHE_VOL=-v $(shell go env GOCACHE):/root/.caches/go-build else TEST_BUILDCACHE_VOL= endif DIST_TAG?=1 DIST_BUILD?=1 DIST_SIGN?=1 ifdef DIST_VERSION DIST_VERSION_ARG=-v "$(DIST_VERSION)" else DIST_VERSION_ARG= endif ifdef DIST_RELEASE_DATE DIST_DATE_ARG=-d "$(DIST_RELEASE_DATE)" else DIST_DATE_ARG= endif ifdef DIST_PRERELEASE DIST_REL_ARG=-r "$(DIST_PRERELEASE)" else DIST_REL_ARG= endif PUB_GIT?=1 PUB_WEBSITE?=1 ifeq ($(PUB_GIT),1) PUB_GIT_ARG=-g else PUB_GIT_ARG= endif ifeq ($(PUB_WEBSITE),1) PUB_WEBSITE_ARG=-w else PUB_WEBSITE_ARG= endif export GO_BUILD_TAG export UI_BUILD_TAG export BUILD_CONTAINER_NAME export GIT_COMMIT export GIT_COMMIT_YEAR export GIT_DIRTY export GOTAGS export GOLDFLAGS # Allow skipping docker build during integration tests in CI since we already # have a built binary ENVOY_INTEG_DEPS?=docker-envoy-integ ifdef SKIP_DOCKER_BUILD ENVOY_INTEG_DEPS=noop endif all: dev-build # used to make integration dependencies conditional noop: ; # dev creates binaries for testing locally - these are put into ./bin dev: dev-build dev-build: mkdir -p bin CGO_ENABLED=0 go install -ldflags "$(GOLDFLAGS)" -tags "$(GOTAGS)" # rm needed due to signature caching (https://apple.stackexchange.com/a/428388) rm -f ./bin/consul cp ${MAIN_GOPATH}/bin/consul ./bin/consul dev-docker-dbg: dev-docker @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" @# 'consul-dbg:local' tag is needed to run the integration tests @# 'consul-dev:latest' is needed by older workflows @docker buildx use default && docker buildx build -t $(CONSUL_COMPAT_TEST_IMAGE)-dbg:local \ --platform linux/$(GOARCH) \ --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ --load \ -f $(CURDIR)/build-support/docker/Consul-Dev-Dbg.dockerfile $(CURDIR)/pkg/bin/ dev-docker: linux dev-build @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" @# 'consul:local' tag is needed to run the integration tests @# 'consul-dev:latest' is needed by older workflows @docker buildx use default && docker buildx build -t 'consul:local' -t '$(CONSUL_DEV_IMAGE)' \ --platform linux/$(GOARCH) \ --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ --load \ -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/ check-remote-dev-image-env: ifndef REMOTE_DEV_IMAGE $(error REMOTE_DEV_IMAGE is undefined: set this image to /:, e.g. hashicorp/consul-k8s-dev:latest) endif remote-docker: check-remote-dev-image-env $(MAKE) GOARCH=amd64 linux $(MAKE) GOARCH=arm64 linux @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building and Pushing Consul Development container - $(REMOTE_DEV_IMAGE)" @if ! docker buildx inspect consul-builder; then \ docker buildx create --name consul-builder --driver docker-container --bootstrap; \ fi; @docker buildx use consul-builder && docker buildx build -t '$(REMOTE_DEV_IMAGE)' \ --platform linux/amd64,linux/arm64 \ --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ --push \ -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/ # In CI, the linux binary will be attached from a previous step at bin/. This make target # should only run in CI and not locally. ci.dev-docker: @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building Consul Development container - $(CI_DEV_DOCKER_IMAGE_NAME)" @docker build $(NOCACHE) $(QUIET) -t '$(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):$(GIT_COMMIT)' \ --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ --label COMMIT_SHA=$(CIRCLE_SHA1) \ --label PULL_REQUEST=$(CIRCLE_PULL_REQUEST) \ --label CIRCLE_BUILD_URL=$(CIRCLE_BUILD_URL) \ $(CI_DEV_DOCKER_WORKDIR) -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile @echo $(DOCKER_PASS) | docker login -u="$(DOCKER_USER)" --password-stdin @echo "Pushing dev image to: https://cloud.docker.com/u/hashicorpdev/repository/docker/hashicorpdev/consul" @docker push $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):$(GIT_COMMIT) ifeq ($(CIRCLE_BRANCH), main) @docker tag $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):$(GIT_COMMIT) $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):latest @docker push $(CI_DEV_DOCKER_NAMESPACE)/$(CI_DEV_DOCKER_IMAGE_NAME):latest endif # linux builds a linux binary compatible with the source platform linux: @mkdir -p ./pkg/bin/linux_$(GOARCH) CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -o ./pkg/bin/linux_$(GOARCH) -ldflags "$(GOLDFLAGS)" -tags "$(GOTAGS)" # dist builds binaries for all platforms and packages them for distribution dist: @$(SHELL) $(CURDIR)/build-support/scripts/release.sh -t '$(DIST_TAG)' -b '$(DIST_BUILD)' -S '$(DIST_SIGN)' $(DIST_VERSION_ARG) $(DIST_DATE_ARG) $(DIST_REL_ARG) cover: cov cov: other-consul dev-build go test -tags '$(GOTAGS)' ./... -coverprofile=coverage.out cd sdk && go test -tags '$(GOTAGS)' ./... -coverprofile=../coverage.sdk.part cd api && go test -tags '$(GOTAGS)' ./... -coverprofile=../coverage.api.part grep -h -v "mode: set" coverage.{sdk,api}.part >> coverage.out rm -f coverage.{sdk,api}.part go tool cover -html=coverage.out test: other-consul dev-build lint test-internal go-mod-tidy: @echo "--> Running go mod tidy" @cd sdk && go mod tidy @cd api && go mod tidy @go mod tidy @cd test/integration/consul-container && go mod tidy @cd test/integration/connect/envoy/test-sds-server && go mod tidy @cd proto-public && go mod tidy @cd internal/tools/proto-gen-rpc-glue && go mod tidy @cd internal/tools/proto-gen-rpc-glue/e2e && go mod tidy @cd internal/tools/proto-gen-rpc-glue/e2e/consul && go mod tidy @cd internal/tools/protoc-gen-consul-rate-limit && go mod tidy test-internal: @echo "--> Running go test" @rm -f test.log exit-code @# Dump verbose output to test.log so we can surface test names on failure but @# hide it from travis as it exceeds their log limits and causes job to be @# terminated (over 4MB and over 10k lines in the UI). We need to output @# _something_ to stop them terminating us due to inactivity... @echo "===================== submodule: sdk =====================" | tee -a test.log cd sdk && { go test -v $(GOTEST_FLAGS) -tags '$(GOTAGS)' ./... 2>&1 ; echo $$? >> ../exit-code ; } | tee -a ../test.log | egrep '^(ok|FAIL|panic:|--- FAIL|--- PASS)' @echo "===================== submodule: api =====================" | tee -a test.log cd api && { go test -v $(GOTEST_FLAGS) -tags '$(GOTAGS)' ./... 2>&1 ; echo $$? >> ../exit-code ; } | tee -a ../test.log | egrep '^(ok|FAIL|panic:|--- FAIL|--- PASS)' @echo "===================== submodule: root =====================" | tee -a test.log { go test -v $(GOTEST_FLAGS) -tags '$(GOTAGS)' ./... 2>&1 ; echo $$? >> exit-code ; } | tee -a test.log | egrep '^(ok|FAIL|panic:|--- FAIL|--- PASS)' @# if everything worked fine then all 3 zeroes will be collapsed to a single zero here @exit_codes="$$(sort -u exit-code)" ; echo "$$exit_codes" > exit-code @echo "Exit code: $$(cat exit-code)" @# This prints all the race report between ====== lines @awk '/^WARNING: DATA RACE/ {do_print=1; print "=================="} do_print==1 {print} /^={10,}/ {do_print=0}' test.log || true @grep -A10 'panic: ' test.log || true @# Prints all the failure output until the next non-indented line - testify @# helpers often output multiple lines for readability but useless if we can't @# see them. Un-intuitive order of matches is necessary. No || true because @# awk always returns true even if there is no match and it breaks non-bash @# shells locally. @awk '/^[^[:space:]]/ {do_print=0} /--- SKIP/ {do_print=1} do_print==1 {print}' test.log @awk '/^[^[:space:]]/ {do_print=0} /--- FAIL/ {do_print=1} do_print==1 {print}' test.log @grep '^FAIL' test.log || true @if [ "$$(cat exit-code)" == "0" ] ; then echo "PASS" ; exit 0 ; else exit 1 ; fi test-race: $(MAKE) GOTEST_FLAGS=-race test-docker: linux go-build-image @# -ti run in the foreground showing stdout @# --rm removes the container once its finished running @# GO_MODCACHE_VOL - args for mapping in the go module cache @# GO_BUILD_CACHE_VOL - args for mapping in the go build cache @# All the env vars are so we pass through all the relevant bits of information @# Needed for running the tests @# We map in our local linux_amd64 bin directory as thats where the linux dep @# target dropped the binary. We could build the binary in the container too @# but that might take longer as caching gets weird @# Lastly we map the source dir here to the /consul workdir @echo "Running tests within a docker container" @docker run -ti --rm \ -e 'GOTEST_FLAGS=$(GOTEST_FLAGS)' \ -e 'GOTAGS=$(GOTAGS)' \ -e 'GIT_COMMIT=$(GIT_COMMIT)' \ -e 'GIT_COMMIT_YEAR=$(GIT_COMMIT_YEAR)' \ -e 'GIT_DIRTY=$(GIT_DIRTY)' \ $(TEST_PARALLELIZATION) \ $(TEST_DOCKER_RESOURCE_CONSTRAINTS) \ $(TEST_MODCACHE_VOL) \ $(TEST_BUILDCACHE_VOL) \ -v $(MAIN_GOPATH)/bin/linux_amd64/:/go/bin \ -v $(shell pwd):/consul \ $(GO_BUILD_TAG) \ make test-internal other-consul: @echo "--> Checking for other consul instances" @if ps -ef | grep 'consul agent' | grep -v grep ; then \ echo "Found other running consul agents. This may affect your tests." ; \ exit 1 ; \ fi lint: -lint-main lint-container-test-deps .PHONY: -lint-main -lint-main: lint-tools @echo "--> Running golangci-lint" @golangci-lint run --build-tags '$(GOTAGS)' && \ (cd api && golangci-lint run --build-tags '$(GOTAGS)') && \ (cd sdk && golangci-lint run --build-tags '$(GOTAGS)') @echo "--> Running golangci-lint (container tests)" @cd test/integration/consul-container && golangci-lint run --build-tags '$(GOTAGS)' @echo "--> Running lint-consul-retry" @lint-consul-retry @echo "--> Running enumcover" @enumcover ./... .PHONY: lint-container-test-deps lint-container-test-deps: @echo "--> Checking container tests for bad dependencies" @cd test/integration/consul-container && ( \ found="$$(go list -m all | grep -c '^github.com/hashicorp/consul ')" ; \ if [[ "$$found" != "0" ]]; then \ echo "test/integration/consul-container: This project should not depend on the root consul module" >&2 ; \ exit 1 ; \ fi \ ) # Build the static web ui inside a Docker container. For local testing only; do not commit these assets. ui: ui-docker # Build the static web ui with yarn. This is the version to commit. .PHONY: ui-regen ui-regen: cd $(CURDIR)/ui && make && cd .. rm -rf $(CURDIR)/agent/uiserver/dist mv $(CURDIR)/ui/packages/consul-ui/dist $(CURDIR)/agent/uiserver/ tools: @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh .PHONY: lint-tools lint-tools: @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh -lint .PHONY: proto-tools proto-tools: @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh -protobuf .PHONY: codegen-tools codegen-tools: @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh -codegen .PHONY: deep-copy deep-copy: codegen-tools @$(SHELL) $(CURDIR)/agent/structs/deep-copy.sh @$(SHELL) $(CURDIR)/agent/proxycfg/deep-copy.sh version: @echo -n "Version: " @$(SHELL) $(CURDIR)/build-support/scripts/version.sh @echo -n "Version + release: " @$(SHELL) $(CURDIR)/build-support/scripts/version.sh -r @echo -n "Version + git: " @$(SHELL) $(CURDIR)/build-support/scripts/version.sh -g @echo -n "Version + release + git: " @$(SHELL) $(CURDIR)/build-support/scripts/version.sh -r -g docker-images: go-build-image ui-build-image go-build-image: @echo "Building Golang build container" @docker build $(NOCACHE) $(QUIET) -t $(GO_BUILD_TAG) - < build-support/docker/Build-Go.dockerfile ui-build-image: @echo "Building UI build container" @docker build $(NOCACHE) $(QUIET) -t $(UI_BUILD_TAG) - < build-support/docker/Build-UI.dockerfile # Builds consul in a docker container and then dumps executable into ./pkg/bin/... consul-docker: go-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh consul ui-docker: ui-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh ui # Build image used to run integration tests locally. docker-envoy-integ: $(MAKE) GOARCH=amd64 linux docker build \ --platform linux/amd64 $(NOCACHE) $(QUIET) \ -t 'consul:local' \ --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ $(CURDIR)/pkg/bin/linux_amd64 \ -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile # Run integration tests. # Use GO_TEST_FLAGS to run specific tests: # make test-envoy-integ GO_TEST_FLAGS="-run TestEnvoy/case-basic" # NOTE: Always uses amd64 images, even when running on M1 macs, to match CI/CD environment. test-envoy-integ: $(ENVOY_INTEG_DEPS) @go test -v -timeout=30m -tags integration $(GO_TEST_FLAGS) ./test/integration/connect/envoy .PHONY: test-compat-integ test-compat-integ: test-compat-integ-setup ifeq ("$(GOTESTSUM_PATH)","") @cd ./test/integration/consul-container && \ go test \ -v \ -timeout=30m \ ./... \ --tags $(GOTAGS) \ --target-image $(CONSUL_COMPAT_TEST_IMAGE) \ --target-version local \ --latest-image $(CONSUL_COMPAT_TEST_IMAGE) \ --latest-version latest else @cd ./test/integration/consul-container && \ gotestsum \ --format=short-verbose \ --debug \ --rerun-fails=3 \ --packages="./..." \ -- \ --tags $(GOTAGS) \ -timeout=30m \ ./... \ --target-image $(CONSUL_COMPAT_TEST_IMAGE) \ --target-version local \ --latest-image $(CONSUL_COMPAT_TEST_IMAGE) \ --latest-version latest endif # NOTE: Use DOCKER_BUILDKIT=0, if docker build fails to resolve consul:local base image .PHONY: test-compat-integ-setup test-compat-integ-setup: dev-docker @docker tag consul-dev:latest $(CONSUL_COMPAT_TEST_IMAGE):local @docker run --rm -t $(CONSUL_COMPAT_TEST_IMAGE):local consul version @# 'consul-envoy:target-version' is needed by compatibility integ test @docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=$(CONSUL_COMPAT_TEST_IMAGE):local --build-arg ENVOY_VERSION=${ENVOY_VERSION} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets .PHONY: test-metrics-integ test-metrics-integ: test-compat-integ-setup @cd ./test/integration/consul-container && \ go test -v -timeout=7m ./test/metrics \ --target-image $(CONSUL_COMPAT_TEST_IMAGE) \ --target-version local \ --latest-image $(CONSUL_COMPAT_TEST_IMAGE) \ --latest-version latest test-connect-ca-providers: @echo "Running /agent/connect/ca tests in verbose mode" @go test -v ./agent/connect/ca @go test -v ./agent/consul -run Vault @go test -v ./agent -run Vault .PHONY: proto proto: proto-tools proto-gen proto-mocks .PHONY: proto-gen proto-gen: proto-tools @$(SHELL) $(CURDIR)/build-support/scripts/protobuf.sh .PHONY: proto-mocks proto-mocks: for dir in $(MOCKED_PB_DIRS) ; do \ cd proto-public && \ rm -f $$dir/mock*.go && \ mockery --dir $$dir --inpackage --all --recursive --log-level trace ; \ done .PHONY: proto-format proto-format: proto-tools @buf format -w .PHONY: proto-lint proto-lint: proto-tools @buf lint @for fn in $$(find proto -name '*.proto'); do \ if [[ "$$fn" = "proto/private/pbsubscribe/subscribe.proto" ]]; then \ continue ; \ elif [[ "$$fn" = "proto/private/pbpartition/partition.proto" ]]; then \ continue ; \ fi ; \ pkg=$$(grep "^package " "$$fn" | sed 's/^package \(.*\);/\1/'); \ if [[ "$$pkg" != hashicorp.consul.internal.* ]]; then \ echo "ERROR: $$fn: is missing 'hashicorp.consul.internal' package prefix: $$pkg" >&2; \ exit 1; \ fi \ done # utility to echo a makefile variable (i.e. 'make print-PROTOC_VERSION') print-% : ; @echo $($*) .PHONY: module-versions # Print a list of modules which can be updated. # Columns are: module current_version date_of_current_version latest_version module-versions: @go list -m -u -f '{{if .Update}} {{printf "%-50v %-40s" .Path .Version}} {{with .Time}} {{ .Format "2006-01-02" -}} {{else}} {{printf "%9s" ""}} {{end}} {{ .Update.Version}} {{end}}' all .PHONY: envoy-library envoy-library: @$(SHELL) $(CURDIR)/build-support/scripts/envoy-library-references.sh .PHONY: envoy-regen envoy-regen: $(info regenerating envoy golden files) @for d in endpoints listeners routes clusters rbac; do \ if [[ -d "agent/xds/testdata/$${d}" ]]; then \ find "agent/xds/testdata/$${d}" -name '*.golden' -delete ; \ fi \ done @go test -tags '$(GOTAGS)' ./agent/xds -update @find "command/connect/envoy/testdata" -name '*.golden' -delete @go test -tags '$(GOTAGS)' ./command/connect/envoy -update # Point your web browser to http://localhost:3000/consul to live render docs from ./website/ .PHONY: docs docs: make -C website .PHONY: help help: $(info available make targets) $(info ----------------------) @grep "^[a-z0-9-][a-z0-9.-]*:" GNUmakefile | cut -d':' -f1 | sort .PHONY: all bin dev dist cov test test-internal cover lint ui tools .PHONY: docker-images go-build-image ui-build-image consul-docker ui-docker .PHONY: version test-envoy-integ