From 7b72ddfb6018ab26fc9314a9948bc0da10a462bc Mon Sep 17 00:00:00 2001 From: "R.B. Boyer" <4903+rboyer@users.noreply.github.com> Date: Mon, 23 May 2022 10:50:45 -0500 Subject: [PATCH] build: wire up remaining 5 helper tools into the same auto-install logic used for protobuf tools (#13184) --- .circleci/config.yml | 5 +- .github/workflows/build.yml | 10 -- GNUmakefile | 63 ++++----- build-support/scripts/devtools.sh | 219 ++++++++++++++++++++++++++++++ build-support/scripts/protobuf.sh | 135 +----------------- 5 files changed, 248 insertions(+), 184 deletions(-) create mode 100644 build-support/scripts/devtools.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a5b13e2497..fa2bb33796 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -185,9 +185,7 @@ jobs: - run: go env - run: name: Install golangci-lint - command: | - download=https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh - wget -O- -q $download | sh -x -s -- -d -b /home/circleci/go/bin v1.46.2 + command: make lint-tools - run: go mod download - run: name: lint @@ -675,7 +673,6 @@ jobs: - attach_workspace: at: ./pkg - run: mv pkg/packages/consul-ui/dist pkg/web_ui # 'make static-assets' looks for the 'pkg/web_ui' path - - run: make tools - run: make static-assets - persist_to_workspace: root: . diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfcb4121e0..b0695fec04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,11 +85,6 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Install project dependencies - run: | - go install github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@38087fe - go install github.com/hashicorp/go-bindata/go-bindata@bf7910a - - name: Setup with node and yarn uses: actions/setup-node@v2 with: @@ -185,11 +180,6 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Install project dependencies - run: | - go install github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@38087fe - go install github.com/hashicorp/go-bindata/go-bindata@bf7910a - - name: Setup with node and yarn uses: actions/setup-node@v2 with: diff --git a/GNUmakefile b/GNUmakefile index 2069c9805f..8a77a54b2d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -2,34 +2,16 @@ # https://www.consul.io/docs/install#compiling-from-source SHELL = bash -GOTOOLS = \ - github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@master \ - github.com/hashicorp/go-bindata/go-bindata@master \ - github.com/vektra/mockery/v2@latest \ - github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.2 \ - github.com/hashicorp/lint-consul-retry@master ### -# BUF_VERSION can be either a valid string for "go install @" +# These version variables can either be a valid string for "go install @" # or the string @DEV to imply use what is currently installed locally. ### +GOLANGCI_LINT_VERSION='v1.46.2' +MOCKERY_VERSION='v2.12.2' BUF_VERSION='v1.4.0' - -### -# PROTOC_GEN_GO_GRPC_VERSION can be either a valid string for "go install @" -# or the string @DEV to imply use what is currently installed locally. -### PROTOC_GEN_GO_GRPC_VERSION="v1.2.0" - -### -# MOG_VERSION can be either a valid string for "go install @" -# or the string @DEV to imply use whatever is currently installed locally. -### MOG_VERSION='v0.3.0' -### -# PROTOC_GO_INJECT_TAG_VERSION can be either a valid string for "go install @" -# or the string @DEV to imply use whatever is currently installed locally. -### PROTOC_GO_INJECT_TAG_VERSION='v1.3.0' GOTAGS ?= @@ -283,33 +265,38 @@ other-consul: exit 1 ; \ fi -lint: - @echo "--> Running go golangci-lint" +lint: lint-tools + @echo "--> Running golangci-lint" @golangci-lint run --build-tags '$(GOTAGS)' && \ (cd api && golangci-lint run --build-tags '$(GOTAGS)') && \ (cd sdk && golangci-lint run --build-tags '$(GOTAGS)') + @echo "--> Running lint-consul-retry" + @lint-consul-retry # If you've run "make ui" manually then this will get called for you. This is # also run as part of the release build script when it verifies that there are no # changes to the UI assets that aren't checked in. -static-assets: +static-assets: bindata-tools @go-bindata-assetfs -pkg uiserver -prefix pkg -o $(ASSETFS_PATH) ./pkg/web_ui/... @go fmt $(ASSETFS_PATH) - # Build the static web ui and build static assets inside a Docker container ui: ui-docker static-assets-docker -tools: proto-tools - @if [[ -d .gotools ]]; then rm -rf .gotools ; fi - @for TOOL in $(GOTOOLS); do \ - echo "=== TOOL: $$TOOL" ; \ - go install -v $$TOOL ; \ - done +tools: + @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh + +.PHONY: lint-tools +lint-tools: + @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh -lint +.PHONY: bindata-tools +bindata-tools: + @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh -bindata + +.PHONY: proto-tools proto-tools: - @$(SHELL) $(CURDIR)/build-support/scripts/protobuf.sh \ - --tools-only + @$(SHELL) $(CURDIR)/build-support/scripts/devtools.sh -protobuf version: @echo -n "Version: " @@ -326,7 +313,7 @@ docker-images: go-build-image ui-build-image go-build-image: @echo "Building Golang build container" - @docker build $(NOCACHE) $(QUIET) --build-arg 'GOTOOLS=$(GOTOOLS)' -t $(GO_BUILD_TAG) - < build-support/docker/Build-Go.dockerfile + @docker build $(NOCACHE) $(QUIET) -t $(GO_BUILD_TAG) - < build-support/docker/Build-Go.dockerfile ui-build-image: @echo "Building UI build container" @@ -376,12 +363,12 @@ endif .PHONY: proto proto: - @$(SHELL) $(CURDIR)/build-support/scripts/protobuf.sh - + @$(SHELL) $(CURDIR)/build-support/scripts/protobuf.sh + .PHONY: proto-format proto-format: proto-tools @buf format -w - + .PHONY: proto-lint proto-lint: proto-tools @buf lint --config proto/buf.yaml --path proto @@ -412,6 +399,6 @@ envoy-regen: @find "command/connect/envoy/testdata" -name '*.golden' -delete @go test -tags '$(GOTAGS)' ./command/connect/envoy -update -.PHONY: all bin dev dist cov test test-internal cover lint ui static-assets tools proto-tools +.PHONY: all bin dev dist cov test test-internal cover lint ui static-assets tools .PHONY: docker-images go-build-image ui-build-image static-assets-docker consul-docker ui-docker .PHONY: version test-envoy-integ diff --git a/build-support/scripts/devtools.sh b/build-support/scripts/devtools.sh new file mode 100644 index 0000000000..4c2efea9da --- /dev/null +++ b/build-support/scripts/devtools.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash + +readonly SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})" +readonly SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" +readonly SOURCE_DIR="$(dirname "$(dirname "${SCRIPT_DIR}")")" +readonly FN_DIR="$(dirname "${SCRIPT_DIR}")/functions" + +source "${SCRIPT_DIR}/functions.sh" + +unset CDPATH + +set -euo pipefail + +usage() { +cat <<-EOF +Usage: ${SCRIPT_NAME} [] + +Description: + Installs various supporting Go tools. + +Options: + -protobuf Just install tools for protobuf. + -lint Just install tools for linting. + -bindata Just install tools for static assets. + -h | --help Print this help text. +EOF +} + +function err_usage { + err "$1" + err "" + err "$(usage)" +} + +function main { + while test $# -gt 0 + do + case "$1" in + -protobuf ) + proto_tools_install + return 0 + ;; + -bindata ) + bindata_install + return 0 + ;; + -lint ) + lint_install + return 0 + ;; + -h | --help ) + usage + return 0 + ;; + esac + done + + # ensure these tools are installed + tools_install +} + +function proto_tools_install { + local protoc_gen_go_version + local protoc_gen_go_grpc_version + local buf_version + local mog_version + local protoc_go_inject_tag_version + + protoc_gen_go_version="$(grep github.com/golang/protobuf go.mod | awk '{print $2}')" + protoc_gen_go_grpc_version="$(make --no-print-directory print-PROTOC_GEN_GO_GRPC_VERSION)" + mog_version="$(make --no-print-directory print-MOG_VERSION)" + protoc_go_inject_tag_version="$(make --no-print-directory print-PROTOC_GO_INJECT_TAG_VERSION)" + buf_version="$(make --no-print-directory print-BUF_VERSION)" + + # echo "go: ${protoc_gen_go_version}" + # echo "mog: ${mog_version}" + # echo "tag: ${protoc_go_inject_tag_version}" + + install_versioned_tool \ + 'buf' \ + 'github.com/bufbuild/buf' \ + "${buf_version}" \ + 'github.com/bufbuild/buf/cmd/buf' + + install_versioned_tool \ + 'protoc-gen-go' \ + 'github.com/golang/protobuf' \ + "${protoc_gen_go_version}" \ + 'github.com/golang/protobuf/protoc-gen-go' + + install_versioned_tool \ + 'protoc-gen-go-grpc' \ + 'google.golang.org/grpc/cmd/protoc-gen-go-grpc' \ + "${protoc_gen_go_grpc_version}" \ + 'google.golang.org/grpc/cmd/protoc-gen-go-grpc' + + install_unversioned_tool \ + protoc-gen-go-binary \ + 'github.com/hashicorp/protoc-gen-go-binary@master' + + install_versioned_tool \ + 'protoc-go-inject-tag' \ + 'github.com/favadi/protoc-go-inject-tag' \ + "${protoc_go_inject_tag_version}" \ + 'github.com/favadi/protoc-go-inject-tag' + + install_versioned_tool \ + 'mog' \ + 'github.com/hashicorp/mog' \ + "${mog_version}" \ + 'github.com/hashicorp/mog' + + return 0 +} + +function lint_install { + local golangci_lint_version + golangci_lint_version="$(make --no-print-directory print-GOLANGCI_LINT_VERSION)" + + install_unversioned_tool \ + 'lint-consul-retry' \ + 'github.com/hashicorp/lint-consul-retry@master' + + install_versioned_tool \ + 'golangci-lint' \ + 'github.com/golangci/golangci-lint' \ + "${golangci_lint_version}" \ + 'github.com/golangci/golangci-lint/cmd/golangci-lint' +} + +function bindata_install { + install_unversioned_tool \ + 'go-bindata' \ + 'github.com/hashicorp/go-bindata/go-bindata@bf7910a' + + install_unversioned_tool \ + 'go-bindata-assetfs' \ + 'github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@38087fe' +} + +function tools_install { + local mockery_version + + mockery_version="$(make --no-print-directory print-MOCKERY_VERSION)" + + install_versioned_tool \ + 'mockery' \ + 'github.com/vektra/mockery/v2' \ + "${mockery_version}" \ + 'github.com/vektra/mockery/v2' + + lint_install + bindata_install + proto_tools_install + + return 0 +} + +function install_unversioned_tool { + local command="$1" + local install="$2" + + if ! command -v "${command}" &>/dev/null ; then + status_stage "installing tool: ${install}" + go install "${install}" + else + debug "skipping tool: ${install} (installed)" + fi + + return 0 +} + +function install_versioned_tool { + local command="$1" + local module="$2" + local version="$3" + local installbase="$4" + + local should_install= + local got + + local expect="${module}@${version}" + local install="${installbase}@${version}" + + if [[ -z "$version" ]]; then + err "cannot install '${command}' no version selected" + return 1 + fi + + if [[ "$version" = "@DEV" ]]; then + if ! command -v "${command}" &>/dev/null ; then + err "dev version of '${command}' requested but not installed" + return 1 + fi + status "skipping tool: ${installbase} (using development version)" + return 0 + fi + + if command -v "${command}" &>/dev/null ; then + got="$(go version -m $(which "${command}") | grep '\bmod\b' | grep "${module}" | + awk '{print $2 "@" $3}')" + if [[ "$expect" != "$got" ]]; then + should_install=1 + fi + else + should_install=1 + fi + + if [[ -n $should_install ]]; then + status_stage "installing tool: ${install}" + go install "${install}" + else + debug "skipping tool: ${install} (installed)" + fi + return 0 +} + +main "$@" +exit $? diff --git a/build-support/scripts/protobuf.sh b/build-support/scripts/protobuf.sh index cac080c7ae..16a7920ab4 100755 --- a/build-support/scripts/protobuf.sh +++ b/build-support/scripts/protobuf.sh @@ -16,13 +16,11 @@ cat <<-EOF Usage: ${SCRIPT_NAME} [] Description: - Installs protoc, various supporting Go tools, and then regenerates all Go - files from protobuf definitions. In addition to running the protoc - generator it will also fixup build tags in the generated code and - regenerate mog outputs and RPC stubs. + Regenerates regenerates all Go files from protobuf definitions. In addition + to running the protoc generator it will also fixup build tags in the + generated code and regenerate mog outputs and RPC stubs. Options: - --tools-only Install all required tools but do not generate outputs. -h | --help Print this help text. EOF } @@ -34,9 +32,6 @@ function err_usage { } function main { - local protoc_version= - local tools_only= - while test $# -gt 0 do case "$1" in @@ -44,20 +39,9 @@ function main { usage return 0 ;; - --tools-only ) - tools_only=1 - shift - ;; esac done - # ensure these tools are installed - proto_tools_install - - if [[ -n $tools_only ]]; then - return 0 - fi - for mod in $(find . -name 'buf.gen.yaml' -exec dirname {} \; | sort) do ( @@ -94,119 +78,6 @@ function main { return 0 } -function proto_tools_install { - local protoc_gen_go_version - local protoc_gen_go_grpc_version - local buf_version - local mog_version - local protoc_go_inject_tag_version - - protoc_gen_go_version="$(grep github.com/golang/protobuf go.mod | awk '{print $2}')" - protoc_gen_go_grpc_version="$(make --no-print-directory print-PROTOC_GEN_GO_GRPC_VERSION)" - mog_version="$(make --no-print-directory print-MOG_VERSION)" - protoc_go_inject_tag_version="$(make --no-print-directory print-PROTOC_GO_INJECT_TAG_VERSION)" - buf_version="$(make --no-print-directory print-BUF_VERSION)" - - # echo "go: ${protoc_gen_go_version}" - # echo "mog: ${mog_version}" - # echo "tag: ${protoc_go_inject_tag_version}" - - install_versioned_tool \ - 'buf' \ - 'github.com/bufbuild/buf' \ - "${buf_version}" \ - 'github.com/bufbuild/buf/cmd/buf' - - install_versioned_tool \ - 'protoc-gen-go' \ - 'github.com/golang/protobuf' \ - "${protoc_gen_go_version}" \ - 'github.com/golang/protobuf/protoc-gen-go' - - install_versioned_tool \ - 'protoc-gen-go-grpc' \ - 'google.golang.org/grpc/cmd/protoc-gen-go-grpc' \ - "${protoc_gen_go_grpc_version}" \ - 'google.golang.org/grpc/cmd/protoc-gen-go-grpc' - - install_unversioned_tool \ - protoc-gen-go-binary \ - 'github.com/hashicorp/protoc-gen-go-binary@master' - - install_versioned_tool \ - 'protoc-go-inject-tag' \ - 'github.com/favadi/protoc-go-inject-tag' \ - "${protoc_go_inject_tag_version}" \ - 'github.com/favadi/protoc-go-inject-tag' - - install_versioned_tool \ - 'mog' \ - 'github.com/hashicorp/mog' \ - "${mog_version}" \ - 'github.com/hashicorp/mog' - - return 0 -} - -function install_unversioned_tool { - local command="$1" - local install="$2" - - if ! command -v "${command}" &>/dev/null ; then - status_stage "installing tool: ${install}" - go install "${install}" - else - debug "skipping tool: ${install} (installed)" - fi - - return 0 -} - -function install_versioned_tool { - local command="$1" - local module="$2" - local version="$3" - local installbase="$4" - - local should_install= - local got - - local expect="${module}@${version}" - local install="${installbase}@${version}" - - if [[ -z "$version" ]]; then - err "cannot install '${command}' no version selected" - return 1 - fi - - if [[ "$version" = "@DEV" ]]; then - if ! command -v "${command}" &>/dev/null ; then - err "dev version of '${command}' requested but not installed" - return 1 - fi - status "skipping tool: ${installbase} (using development version)" - return 0 - fi - - if command -v "${command}" &>/dev/null ; then - got="$(go version -m $(which "${command}") | grep '\bmod\b' | grep "${module}" | - awk '{print $2 "@" $3}')" - if [[ "$expect" != "$got" ]]; then - should_install=1 - fi - else - should_install=1 - fi - - if [[ -n $should_install ]]; then - status_stage "installing tool: ${install}" - go install "${install}" - else - debug "skipping tool: ${install} (installed)" - fi - return 0 -} - function postprocess_protobuf_code { local proto_path="${1:-}" if [[ -z "${proto_path}" ]]; then