build: install mog and execute it during protobuf compilation (#12647)

- also import replace isn't needed anymore

.circleci/config.yml

@@ -237,11 +237,7 @@ jobs:
             rm protoc-*.zip
       - run:
           name: Install protobuf
-          command: |
-            goproto_version=$(go list -m github.com/golang/protobuf | awk '{print $2}')
-            go install -v github.com/hashicorp/protoc-gen-go-binary@master
-            go install -v github.com/favadi/protoc-go-inject-tag@v1.3.0
-            go install -v github.com/golang/protobuf/protoc-gen-go@${goproto_version}
+          command: make proto-tools
 
       - run:
           command: make --always-make proto

GNUmakefile

@@ -2,19 +2,22 @@
 # https://www.consul.io/docs/install#compiling-from-source
 
 SHELL = bash
-PROTOC_VERSION=3.12.3
-GOPROTOVERSION?=$(shell grep github.com/golang/protobuf go.mod | awk '{print $$2}')
 GOTOOLS = \
 	github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@master \
 	github.com/hashicorp/go-bindata/go-bindata@master \
 	golang.org/x/tools/cmd/cover@master \
 	golang.org/x/tools/cmd/stringer@master \
-	github.com/golang/protobuf/protoc-gen-go@$(GOPROTOVERSION) \
-	github.com/hashicorp/protoc-gen-go-binary@master \
 	github.com/vektra/mockery/cmd/mockery@master \
 	github.com/golangci/golangci-lint/cmd/golangci-lint@v1.40.1 \
-	github.com/hashicorp/lint-consul-retry@master \
-	github.com/favadi/protoc-go-inject-tag@v1.3.0
+	github.com/hashicorp/lint-consul-retry@master
+
+PROTOC_VERSION=3.12.3
+GOPROTOVERSION?=$(shell grep github.com/golang/protobuf go.mod | awk '{print $$2}')
+GOPROTOTOOLS = \
+	github.com/golang/protobuf/protoc-gen-go@$(GOPROTOVERSION) \
+	github.com/hashicorp/protoc-gen-go-binary@master \
+	github.com/favadi/protoc-go-inject-tag@v1.3.0 \
+	github.com/hashicorp/mog@v0.1.1
 
 GOTAGS ?=
 GOPATH=$(shell go env GOPATH)
@@ -287,13 +290,20 @@ static-assets:
 # Build the static web ui and build static assets inside a Docker container
 ui: ui-docker static-assets-docker
 
-tools:
+tools: proto-tools
 	@if [[ -d .gotools ]]; then rm -rf .gotools ; fi
 	@for TOOL in $(GOTOOLS); do \
 		echo "=== TOOL: $$TOOL" ; \
 		go install -v $$TOOL ; \
 	done
 
+proto-tools:
+	@if [[ -d .gotools ]]; then rm -rf .gotools ; fi
+	@for TOOL in $(GOPROTOTOOLS); do \
+		echo "=== TOOL: $$TOOL" ; \
+		go install -v $$TOOL ; \
+	done
+
 version:
 	@echo -n "Version:                    "
 	@$(SHELL) $(CURDIR)/build-support/scripts/version.sh
@@ -358,7 +368,7 @@ proto: protoc-check $(PROTOGOFILES) $(PROTOGOBINFILES)
 	@echo "Generated all protobuf Go files"
 
 %.pb.go %.pb.binary.go: %.proto
-	@$(SHELL) $(CURDIR)/build-support/scripts/proto-gen.sh --grpc --import-replace "$<"
+	@$(SHELL) $(CURDIR)/build-support/scripts/proto-gen.sh --grpc "$<"
 
 # utility to echo a makefile variable (i.e. 'make print-PROTOC_VERSION')
 print-%  : ; @echo $($*)
@@ -385,6 +395,6 @@ envoy-regen:
 	@find "command/connect/envoy/testdata" -name '*.golden' -delete
 	@go test -tags '$(GOTAGS)' ./command/connect/envoy -update
 
-.PHONY: all bin dev dist cov test test-internal cover lint ui static-assets tools
+.PHONY: all bin dev dist cov test test-internal cover lint ui static-assets tools proto-tools protoc-check
 .PHONY: docker-images go-build-image ui-build-image static-assets-docker consul-docker ui-docker
 .PHONY: version proto test-envoy-integ

build-support/scripts/proto-gen.sh

@@ -37,7 +37,6 @@ function err_usage {
 
 function main {
    local -i grpc=0
-   local -i imp_replace=0
    local    proto_path=
 
    while test $# -gt 0
@@ -51,10 +50,6 @@ function main {
             grpc=1
             shift
             ;;
-         --import-replace )
-            imp_replace=1
-            shift
-            ;;
          * )
             proto_path="$1"
             shift
@@ -73,13 +68,10 @@ function main {
    local golang_proto_path=$(go list -f '{{ .Dir }}' -m github.com/golang/protobuf)
    local golang_proto_mod_path=$(sed -e 's,\(.*\)github.com.*,\1,' <<< "${golang_proto_path}")
 
-
-   local golang_proto_imp_replace="Mgoogle/protobuf/timestamp.proto=github.com/golang/protobuf/ptypes/timestamp"
-   golang_proto_imp_replace="${golang_proto_imp_replace},Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration"
-
    local proto_go_path=${proto_path%%.proto}.pb.go
    local proto_go_bin_path=${proto_path%%.proto}.pb.binary.go
    local proto_go_rpcglue_path=${proto_path%%.proto}.rpcglue.pb.go
+   local mog_input_path="$(dirname "${proto_path}")"
 
    local go_proto_out="paths=source_relative"
    if is_set "${grpc}"
@@ -87,23 +79,20 @@ function main {
       go_proto_out="${go_proto_out},plugins=grpc"
    fi
 
-   if is_set "${imp_replace}"
-   then
-      go_proto_out="${go_proto_out},${golang_proto_imp_replace}"
-   fi
-
    if test -n "${go_proto_out}"
    then
       go_proto_out="${go_proto_out}:"
    fi
 
+   rm -f "${proto_go_path}" ${proto_go_bin_path}" ${proto_go_rpcglue_path}" "${mog_input_path}/*.gen.go"
+
    # How we run protoc probably needs some documentation.
    #
    # This is the path to where
    #  -I="${golang_proto_path}/protobuf" \
    local -i ret=0
-   status_stage "Generating ${proto_path} into ${proto_go_path} and ${proto_go_bin_path}"
-    echo "debug_run protoc \
+   status_stage "Generating ${proto_path} into ${proto_go_path} and ${proto_go_bin_path} ${mog_input_path}/*.gen.go"
+   echo "debug_run protoc \
           -I=\"${golang_proto_path}\" \
           -I=\"${golang_proto_mod_path}\" \
           -I=\"${SOURCE_DIR}\" \
@@ -133,13 +122,13 @@ function main {
       return 1
    fi
 
-   echo "debug_run protoc \
-         -I=\"${golang_proto_path}\" \
-         -I=\"${golang_proto_mod_path}\" \
-         -I=\"${SOURCE_DIR}\" \
-         --go_out=\"${go_proto_out}${SOURCE_DIR}\" \
-         --go-binary_out=\"${SOURCE_DIR}\" \
-         \"${proto_path}\""
+   debug_run mog -source ./${mog_input_path} -tags ${GOTAGS} -ignore-package-load-errors
+
+   if test $? -ne 0
+   then
+      err "Failed to generate mog outputs from ${mog_input_path}"
+      return 1
+   fi
 
    BUILD_TAGS=$(sed -e '/^[[:space:]]*$/,$d' < "${proto_path}" | grep '// +build')
    if test -n "${BUILD_TAGS}"