proto: Allow reproducible proto generation.

Current way of setting prometheus project to generate repo is tedious.
If you just go install packages described in README.md
it ends up with those (non buildable code)
```
../../../prompb/remote.pb.go:61:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:109:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:164:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:214:27: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:284:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
```
The issue is that the guide presented in proto folder is asking to
install some GO packages. It does not mention from what version and
there is strong mismatch between those used to generate proto and those
that are vendored and used to run the code later...
The particular outdated packages in vendor are:
`github.com/gogo/protobuf` and `google.golang.org/grpc`.

Just updating vendor would be not enough, as it would only tmp fix.
I propose adding script that will parse version from vendor.json and install correct version.

Changes:
- Added script that uses packages in *exactly* the same version the Go code is build against later.
- Regenerated proto with those pinned deps.
- Updated REAMDE.md
- Added `make proto`Current way of setting prometheus project to generate code from proto files is tedious.
If you just go install packages described in README.md, it ends up with this (non buildable code)
```
../../../prompb/remote.pb.go:61:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:109:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:164:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:214:27: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
../../../prompb/remote.pb.go:284:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo
```
The issue is that the guide presented in proto folder is asking to install some GO packages. It does not mention from what version and there is strong mismatch between those used to generate proto and those that are vendored and used to run the code later...
The particular outdated packages in vendor are:
`github.com/gogo/protobuf` and `google.golang.org/grpc`.

Just updating vendor would be not enough, as it would only tmp fix.
I propose adding script that will parse version from vendor.json and install correct version.

Changes:
- Added script that uses packages in *exactly* the same version the Go code is build against later.
- Regenerated proto with those pinned deps.
- Updated REAMDE.md
- Added `make proto`

Signed-off-by: Bartek Plotka <bwplotka@gmail.com>

Makefile.common

@@ -123,6 +123,11 @@ common-docker-tag-latest:
 promu:
 	GOOS= GOARCH= $(GO) get -u github.com/prometheus/promu
 
+.PHONY: proto
+proto:
+	@echo ">> generating code from proto files"
+	@./scripts/genproto.sh
+
 .PHONY: $(STATICCHECK)
 $(STATICCHECK):
 	GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck

prompb/README.md

@@ -2,13 +2,8 @@ The compiled protobufs are version controlled and you won't normally need to
 re-compile them when building Prometheus.
 
 If however you have modified the defs and do need to re-compile, run
-`./scripts/genproto.sh` from the parent dir.
+`make proto` from the parent dir.
 
-In order for the script to run, you'll need `protoc` (version 3.5) in your
+In order for the script to run, you'll need `protoc` (version 3.5.1) in your
-PATH, and the following Go packages installed:
+PATH.
 
-- github.com/gogo/protobuf
-- github.com/gogo/protobuf/protoc-gen-gogofast
-- github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/
-- github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
-- golang.org/x/tools/cmd/goimports

prompb/remote.pb.go

@@ -330,24 +330,6 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
-func encodeFixed64Remote(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Remote(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintRemote(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)

prompb/rpc.pb.go

@@ -11,12 +11,10 @@ import _ "google.golang.org/genproto/googleapis/api/annotations"
 
 import time "time"
 
-import (
+import context "golang.org/x/net/context"
-	context "golang.org/x/net/context"
+import grpc "google.golang.org/grpc"
-	grpc "google.golang.org/grpc"
-)
 
-import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types"
+import types "github.com/gogo/protobuf/types"
 
 import io "io"
 
@@ -338,8 +336,8 @@ func (m *SeriesDeleteRequest) MarshalTo(dAtA []byte) (int, error) {
 	if m.MinTime != nil {
 		dAtA[i] = 0xa
 		i++
-		i = encodeVarintRpc(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.MinTime)))
+		i = encodeVarintRpc(dAtA, i, uint64(types.SizeOfStdTime(*m.MinTime)))
-		n1, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MinTime, dAtA[i:])
+		n1, err := types.StdTimeMarshalTo(*m.MinTime, dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
@@ -348,8 +346,8 @@ func (m *SeriesDeleteRequest) MarshalTo(dAtA []byte) (int, error) {
 	if m.MaxTime != nil {
 		dAtA[i] = 0x12
 		i++
-		i = encodeVarintRpc(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxTime)))
+		i = encodeVarintRpc(dAtA, i, uint64(types.SizeOfStdTime(*m.MaxTime)))
-		n2, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MaxTime, dAtA[i:])
+		n2, err := types.StdTimeMarshalTo(*m.MaxTime, dAtA[i:])
 		if err != nil {
 			return 0, err
 		}
@@ -388,24 +386,6 @@ func (m *SeriesDeleteResponse) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
-func encodeFixed64Rpc(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Rpc(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintRpc(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -450,11 +430,11 @@ func (m *SeriesDeleteRequest) Size() (n int) {
 	var l int
 	_ = l
 	if m.MinTime != nil {
-		l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.MinTime)
+		l = types.SizeOfStdTime(*m.MinTime)
 		n += 1 + l + sovRpc(uint64(l))
 	}
 	if m.MaxTime != nil {
-		l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxTime)
+		l = types.SizeOfStdTime(*m.MaxTime)
 		n += 1 + l + sovRpc(uint64(l))
 	}
 	if len(m.Matchers) > 0 {
@@ -792,7 +772,7 @@ func (m *SeriesDeleteRequest) Unmarshal(dAtA []byte) error {
 			if m.MinTime == nil {
 				m.MinTime = new(time.Time)
 			}
-			if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.MinTime, dAtA[iNdEx:postIndex]); err != nil {
+			if err := types.StdTimeUnmarshal(m.MinTime, dAtA[iNdEx:postIndex]); err != nil {
 				return err
 			}
 			iNdEx = postIndex
@@ -825,7 +805,7 @@ func (m *SeriesDeleteRequest) Unmarshal(dAtA []byte) error {
 			if m.MaxTime == nil {
 				m.MaxTime = new(time.Time)
 			}
-			if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.MaxTime, dAtA[iNdEx:postIndex]); err != nil {
+			if err := types.StdTimeUnmarshal(m.MaxTime, dAtA[iNdEx:postIndex]); err != nil {
 				return err
 			}
 			iNdEx = postIndex

prompb/rpc.pb.gw.go

@@ -58,7 +58,7 @@ func request_Admin_DeleteSeries_0(ctx context.Context, marshaler runtime.Marshal
 	var protoReq SeriesDeleteRequest
 	var metadata runtime.ServerMetadata
 
-	if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && err != io.EOF {
+	if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil {
 		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
 	}
 

prompb/types.pb.go

@@ -7,6 +7,8 @@ import proto "github.com/gogo/protobuf/proto"
 import fmt "fmt"
 import math "math"
 
+import binary "encoding/binary"
+
 import io "io"
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -229,7 +231,8 @@ func (m *Sample) MarshalTo(dAtA []byte) (int, error) {
 	if m.Value != 0 {
 		dAtA[i] = 0x9
 		i++
-		i = encodeFixed64Types(dAtA, i, uint64(math.Float64bits(float64(m.Value))))
+		binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.Value))))
+		i += 8
 	}
 	if m.Timestamp != 0 {
 		dAtA[i] = 0x10
@@ -415,24 +418,6 @@ func (m *ReadHints) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
-func encodeFixed64Types(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Types(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -584,15 +569,8 @@ func (m *Sample) Unmarshal(dAtA []byte) error {
 			if (iNdEx + 8) > l {
 				return io.ErrUnexpectedEOF
 			}
+			v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:]))
 			iNdEx += 8
-			v = uint64(dAtA[iNdEx-8])
-			v |= uint64(dAtA[iNdEx-7]) << 8
-			v |= uint64(dAtA[iNdEx-6]) << 16
-			v |= uint64(dAtA[iNdEx-5]) << 24
-			v |= uint64(dAtA[iNdEx-4]) << 32
-			v |= uint64(dAtA[iNdEx-3]) << 40
-			v |= uint64(dAtA[iNdEx-2]) << 48
-			v |= uint64(dAtA[iNdEx-1]) << 56
 			m.Value = float64(math.Float64frombits(v))
 		case 2:
 			if wireType != 0 {

scripts/genproto.sh

@@ -10,11 +10,31 @@ if ! [[ "$0" =~ "scripts/genproto.sh" ]]; then
 	exit 255
 fi
 
-if ! [[ $(protoc --version) =~ "3.5" ]]; then
+if ! [[ $(protoc --version) =~ "3.5.1" ]]; then
-	echo "could not find protoc 3.5.x, is it installed + in PATH?"
+	echo "could not find protoc 3.5.1, is it installed + in PATH?"
 	exit 255
 fi
 
+# Exact version of plugins to build.
+PROTOC_GEN_GOGOFAST_SHA="971cbfd2e72b513a28c74af7462aee0800248d69"
+PROTOC_GEN_GRPC_ECOSYSTEM_SHA="e4b8a938efae14de11fd97311e873e989896348c"
+
+echo "installing plugins"
+go install "golang.org/x/tools/cmd/goimports"
+go get -d -u "github.com/gogo/protobuf/protoc-gen-gogo"
+pushd ${GOPATH}/src/github.com/gogo/protobuf
+    git reset --hard "${PROTOC_GEN_GOGOFAST_SHA}"
+    go install "github.com/gogo/protobuf/protoc-gen-gogofast"
+popd
+
+go get -d -u "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway"
+go get -d -u "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger"
+pushd ${GOPATH}/src/github.com/grpc-ecosystem/grpc-gateway
+    git reset --hard "${PROTOC_GEN_GRPC_ECOSYSTEM_SHA}"
+    go install "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway"
+    go install "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger"
+popd
+
 PROM_ROOT="${GOPATH}/src/github.com/prometheus/prometheus"
 PROM_PATH="${PROM_ROOT}/prompb"
 GOGOPROTO_ROOT="${GOPATH}/src/github.com/gogo/protobuf"
@@ -40,7 +60,7 @@ for dir in ${DIRS}; do
 			rpc.proto
 		mv ../documentation/dev/api/rpc.swagger.json ../documentation/dev/api/swagger.json
 		
-		sed -i.bak -E 's/import _ \"gogoproto\"//g' *.pb.go
+		sed -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' *.pb.go
 		sed -i.bak -E 's/import _ \"google\/protobuf\"//g' *.pb.go
 		sed -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' *.go
 		rm -f *.bak