Merge pull request #2786 from tomwilkie/report-error-in-remote-write

Remote write: read first line of response and include it in the error.

storage/remote/client.go

@@ -14,8 +14,10 @@
 package remote
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"time"
@@ -31,6 +33,8 @@ import (
 	"github.com/prometheus/prometheus/util/httputil"
 )
 
+const maxErrMsgLen = 256
+
 // Client allows reading and writing from/to a remote HTTP endpoint.
 type Client struct {
 	index   int // Used to differentiate metrics.
@@ -117,7 +121,12 @@ func (c *Client) Store(samples model.Samples) error {
 	defer httpResp.Body.Close()
 
 	if httpResp.StatusCode/100 != 2 {
-		err = fmt.Errorf("server returned HTTP status %s", httpResp.Status)
+		scanner := bufio.NewScanner(io.LimitReader(httpResp.Body, maxErrMsgLen))
+		line := ""
+		if scanner.Scan() {
+			line = scanner.Text()
+		}
+		err = fmt.Errorf("server returned HTTP status %s: %s", httpResp.Status, line)
 	}
 	if httpResp.StatusCode/100 == 5 {
 		return recoverableError{err}

storage/remote/client_test.go

@@ -19,6 +19,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -26,6 +27,8 @@ import (
 	"github.com/prometheus/prometheus/config"
 )
 
+var longErrMessage = strings.Repeat("error message", maxErrMsgLen)
+
 func TestStoreHTTPErrorHandling(t *testing.T) {
 	tests := []struct {
 		code int
@@ -37,22 +40,22 @@ func TestStoreHTTPErrorHandling(t *testing.T) {
 		},
 		{
 			code: 300,
-			err:  fmt.Errorf("server returned HTTP status 300 Multiple Choices"),
+			err:  fmt.Errorf("server returned HTTP status 300 Multiple Choices: " + longErrMessage[:maxErrMsgLen]),
 		},
 		{
 			code: 404,
-			err:  fmt.Errorf("server returned HTTP status 404 Not Found"),
+			err:  fmt.Errorf("server returned HTTP status 404 Not Found: " + longErrMessage[:maxErrMsgLen]),
 		},
 		{
 			code: 500,
-			err:  recoverableError{fmt.Errorf("server returned HTTP status 500 Internal Server Error")},
+			err:  recoverableError{fmt.Errorf("server returned HTTP status 500 Internal Server Error: " + longErrMessage[:maxErrMsgLen])},
 		},
 	}
 
 	for i, test := range tests {
 		server := httptest.NewServer(
 			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				http.Error(w, "test error", test.code)
+				http.Error(w, longErrMessage, test.code)
 			}),
 		)
 
@@ -68,7 +71,7 @@ func TestStoreHTTPErrorHandling(t *testing.T) {
 
 		err = c.Store(nil)
 		if !reflect.DeepEqual(err, test.err) {
-			t.Fatalf("%d. Unexpected error; want %v, got %v", i, test.err, err)
+			t.Errorf("%d. Unexpected error; want %v, got %v", i, test.err, err)
 		}
 
 		server.Close()