From c6be3b525b5630feb9e5c43640bc5c209c93fe5d Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 12 Apr 2019 15:17:13 -0400 Subject: [PATCH] Build System Fixes for Go Modules (#5655) * Docker based builds can now use the module cache * Simplify building the consul-dev docker image. * Make sure to pull the latest consul image. * Allow selecting base image version for the dev image --- GNUmakefile | 8 +++-- build-support/docker/Consul-Dev.dockerfile | 35 ++-------------------- build-support/functions/10-util.sh | 34 ++++++++++++++++++--- build-support/functions/20-build.sh | 34 +++++++++++++++++++-- 4 files changed, 71 insertions(+), 40 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index f4497fd62b..71fda62b43 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -44,6 +44,7 @@ CONSUL_DEV_IMAGE?=consul-dev GO_BUILD_TAG?=consul-build-go UI_BUILD_TAG?=consul-build-ui BUILD_CONTAINER_NAME?=consul-builder +CONSUL_IMAGE_VERSION?=latest DIST_TAG?=1 DIST_BUILD?=1 @@ -114,8 +115,11 @@ dev: changelogfmt dev-build dev-build: @$(SHELL) $(CURDIR)/build-support/scripts/build-local.sh -o $(GOOS) -a $(GOARCH) -dev-docker: go-build-image - @docker build -t '$(CONSUL_DEV_IMAGE)' --build-arg 'GIT_COMMIT=$(GIT_COMMIT)' --build-arg 'GIT_DIRTY=$(GIT_DIRTY)' --build-arg 'GIT_DESCRIBE=$(GIT_DESCRIBE)' --build-arg 'CONSUL_BUILD_IMAGE=$(GO_BUILD_TAG)' -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile '$(CURDIR)' +dev-docker: linux + @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" + @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null + @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" + @docker build $(NOCACHE) $(QUIET) -t '$(CONSUL_DEV_IMAGE)' --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) $(CURDIR)/pkg/bin/linux_amd64 -f $(CURDIR)/build-support/docker/Consul-Dev.dockerfile changelogfmt: @echo "--> Making [GH-xxxx] references clickable..." diff --git a/build-support/docker/Consul-Dev.dockerfile b/build-support/docker/Consul-Dev.dockerfile index 54751c12de..fdde521b4f 100644 --- a/build-support/docker/Consul-Dev.dockerfile +++ b/build-support/docker/Consul-Dev.dockerfile @@ -1,32 +1,3 @@ -ARG CONSUL_BUILD_IMAGE -FROM ${CONSUL_BUILD_IMAGE}:latest as builder -# FROM golang:latest as builder -ARG GIT_COMMIT -ARG GIT_DIRTY -ARG GIT_DESCRIBE -# WORKDIR /go/src/github.com/hashicorp/consul -ENV CONSUL_DEV=1 -ENV COLORIZE=0 - -# Cache modules separately from more frequently edited source files. -# -# The trick is taken from [https://medium.com/@pliutau/docker-and-go-modules-4265894f9fc#6622] -# -# We copy the modules files in first since they are less likely to change frequently -# and the population of the go mod cache will be invalidated less frequently. -COPY go.mod . -COPY go.sum . -RUN mkdir -p api sdk -COPY api/go.mod api -COPY api/go.sum api -COPY sdk/go.mod sdk -COPY sdk/go.sum sdk -RUN go mod download - -# Add the rest of the code. -ADD . /consul/ -RUN make dev - -FROM consul:latest - -COPY --from=builder /go/bin/consul /bin +ARG CONSUL_IMAGE_VERSION=latest +FROM consul:${CONSUL_IMAGE_VERSION} +COPY consul /go diff --git a/build-support/functions/10-util.sh b/build-support/functions/10-util.sh index 061f33a233..dcfc2561b3 100644 --- a/build-support/functions/10-util.sh +++ b/build-support/functions/10-util.sh @@ -959,7 +959,7 @@ function shasum_directory { return $ret } - function ui_version { +function ui_version { # Arguments: # $1 - path to index.html # @@ -977,8 +977,9 @@ function shasum_directory { local ui_version="$(grep '$' "$1" | sed 's/^$/\1/')" || return 1 echo "$ui_version" return 0 - } - function ui_logo_type { +} + +function ui_logo_type { # Arguments: # $1 - path to index.html # @@ -1004,4 +1005,29 @@ function shasum_directory { echo "oss" fi return 0 - } +} + +function go_mod_assert { + # Returns: + # 0 - success + # * - failure + # + # Notes: will ensure all the necessary go modules are cached + # and if the CONSUL_MOD_VERIFY env var is set will force + # reverification of all modules. + if ! go mod download >/dev/null + then + err "ERROR: Failed to populate the go module cache" + return 1 + fi + + if is_set "${CONSUL_MOD_VERIFY}" + then + if ! go mod verify + then + err "ERROR: Failed to verify go module checksums" + return 1 + fi + fi + return 0 +} diff --git a/build-support/functions/20-build.sh b/build-support/functions/20-build.sh index 1afd641c28..50f6701a8a 100644 --- a/build-support/functions/20-build.sh +++ b/build-support/functions/20-build.sh @@ -254,7 +254,6 @@ function build_consul { fi pushd ${sdir} > /dev/null - status "Creating the Go Build Container with image: ${image_name}" if is_set "${CONSUL_DEV}" then if test -z "${XC_OS}" @@ -275,7 +274,38 @@ function build_consul { extra_dir="${extra_dir_name}/" fi - local container_id=$(docker create -it -e CGO_ENABLED=0 ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !freebsd/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/bin/${extra_dir}{{.OS}}_{{.Arch}}/consul" -tags="${GOTAGS}") + # figure out if the compiler supports modules + local use_modules=0 + if go help modules >/dev/null 2>&1 + then + use_modules=1 + elif test -n "${GO111MODULE}" + then + use_modules=1 + fi + + local volume_mount= + if is_set "${use_modules}" + then + status "Ensuring Go modules are up to date" + # ensure our go module cache is correct + go_mod_assert || return 1 + # setup to bind mount our hosts module cache into the container + volume_mount="--mount=type=bind,source=${MAIN_GOPATH}/pkg/mod,target=/go/pkg/mod" + fi + + status "Creating the Go Build Container with image: ${image_name}" + local container_id=$(docker create -it \ + ${volume_mount} \ + -e CGO_ENABLED=0 \ + ${image_name} \ + gox \ + -os="${XC_OS}" \ + -arch="${XC_ARCH}" \ + -osarch="!darwin/arm !freebsd/arm !darwin/arm64" \ + -ldflags "${GOLDFLAGS}" \ + -output "pkg/bin/${extra_dir}{{.OS}}_{{.Arch}}/consul" \ + -tags="${GOTAGS}") ret=$? if test $ret -eq 0