diff --git a/web/web.go b/web/web.go index e965c1099..5e43cd357 100644 --- a/web/web.go +++ b/web/web.go @@ -352,6 +352,11 @@ func (h *Handler) testReady(f http.HandlerFunc) http.HandlerFunc { } } +// Checks if server is ready, calls f if it is, returns 503 if it is not. +func (h *Handler) testReadyHandler(f http.Handler) http.HandlerFunc { + return h.testReady(f.ServeHTTP) +} + // Quit returns the receive-only quit channel. func (h *Handler) Quit() <-chan struct{} { return h.quitCh @@ -398,6 +403,8 @@ func (h *Handler) Run(ctx context.Context) error { return err } + hhFunc := h.testReadyHandler(hh) + operationName := nethttp.OperationNameFunc(func(r *http.Request) string { return fmt.Sprintf("%s %s", r.Method, r.URL.Path) }) @@ -417,7 +424,7 @@ func (h *Handler) Run(ctx context.Context) error { mux.Handle(apiPath+"/", http.StripPrefix(apiPath, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { setCORS(w) - hh.ServeHTTP(w, r) + hhFunc(w, r) }), )) diff --git a/web/web_test.go b/web/web_test.go index eb71c73a0..22bb5a906 100644 --- a/web/web_test.go +++ b/web/web_test.go @@ -15,12 +15,16 @@ package web import ( "context" + "io/ioutil" "net/http" "net/url" + "os" + "strings" "testing" "time" "github.com/prometheus/prometheus/storage/tsdb" + libtsdb "github.com/prometheus/tsdb" ) func TestGlobalURL(t *testing.T) { @@ -75,6 +79,16 @@ func TestGlobalURL(t *testing.T) { func TestReadyAndHealthy(t *testing.T) { t.Parallel() + dbDir, err := ioutil.TempDir("", "tsdb-ready") + if err != nil { + t.Fatalf("Unexpected error creating a tmpDir: %s", err) + } + defer os.RemoveAll(dbDir) + db, err := libtsdb.Open(dbDir, nil, nil, nil) + if err != nil { + t.Fatalf("Unexpected error opening empty dir: %s", err) + } + opts := &Options{ ListenAddress: ":9090", ReadTimeout: 30 * time.Second, @@ -87,6 +101,8 @@ func TestReadyAndHealthy(t *testing.T) { Notifier: nil, RoutePrefix: "/", MetricsPath: "/metrics/", + EnableAdminAPI: true, + TSDB: func() *libtsdb.DB { return db }, } opts.Flags = map[string]string{} @@ -122,6 +138,22 @@ func TestReadyAndHealthy(t *testing.T) { t.Fatalf("Path /version with server unready test, Expected status 503 got: %s", resp.Status) } + resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/snapshot", "", strings.NewReader("")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("Path /api/v2/admin/tsdb/snapshot with server unready test, Expected status 503 got: %s", resp.Status) + } + + resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("Path /api/v2/admin/tsdb/delete_series with server unready test, Expected status 503 got: %s", resp.Status) + } + // Set to ready. webHandler.Ready() @@ -148,10 +180,36 @@ func TestReadyAndHealthy(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("Path /version with server ready test, Expected status 200 got: %s", resp.Status) } + + resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/snapshot", "", strings.NewReader("")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("Path /api/v2/admin/tsdb/snapshot with server unready test, Expected status 503 got: %s", resp.Status) + } + + resp, err = http.Post("http://localhost:9090/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("Path /api/v2/admin/tsdb/delete_series with server unready test, Expected status 503 got: %s", resp.Status) + } } func TestRoutePrefix(t *testing.T) { t.Parallel() + dbDir, err := ioutil.TempDir("", "tsdb-ready") + if err != nil { + t.Fatalf("Unexpected error creating a tmpDir: %s", err) + } + defer os.RemoveAll(dbDir) + db, err := libtsdb.Open(dbDir, nil, nil, nil) + if err != nil { + t.Fatalf("Unexpected error opening empty dir: %s", err) + } + opts := &Options{ ListenAddress: ":9091", ReadTimeout: 30 * time.Second, @@ -164,6 +222,8 @@ func TestRoutePrefix(t *testing.T) { Notifier: nil, RoutePrefix: "/prometheus", MetricsPath: "/prometheus/metrics", + EnableAdminAPI: true, + TSDB: func() *libtsdb.DB { return db }, } opts.Flags = map[string]string{} @@ -185,7 +245,7 @@ func TestRoutePrefix(t *testing.T) { t.Fatalf("Unexpected HTTP error %s", err) } if resp.StatusCode != http.StatusOK { - t.Fatalf("Path "+opts.RoutePrefix+"/-/healthy with server unready test, Expected status 200 got: %s", resp.Status) + t.Fatalf("Path %s/-/healthy with server unready test, Expected status 200 got: %s", opts.RoutePrefix, resp.Status) } resp, err = http.Get("http://localhost:9091" + opts.RoutePrefix + "/-/ready") @@ -193,7 +253,7 @@ func TestRoutePrefix(t *testing.T) { t.Fatalf("Unexpected HTTP error %s", err) } if resp.StatusCode != http.StatusServiceUnavailable { - t.Fatalf("Path "+opts.RoutePrefix+"/-/ready with server unready test, Expected status 503 got: %s", resp.Status) + t.Fatalf("Path %s/-/ready with server unready test, Expected status 503 got: %s", opts.RoutePrefix, resp.Status) } resp, err = http.Get("http://localhost:9091" + opts.RoutePrefix + "/version") @@ -201,7 +261,23 @@ func TestRoutePrefix(t *testing.T) { t.Fatalf("Unexpected HTTP error %s", err) } if resp.StatusCode != http.StatusServiceUnavailable { - t.Fatalf("Path "+opts.RoutePrefix+"/version with server unready test, Expected status 503 got: %s", resp.Status) + t.Fatalf("Path %s/version with server unready test, Expected status 503 got: %s", opts.RoutePrefix, resp.Status) + } + + resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/snapshot", "", strings.NewReader("")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("Path %s/api/v2/admin/tsdb/snapshot with server unready test, Expected status 503 got: %s", opts.RoutePrefix, resp.Status) + } + + resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("Path %s/api/v2/admin/tsdb/delete_series with server unready test, Expected status 503 got: %s", opts.RoutePrefix, resp.Status) } // Set to ready. @@ -230,4 +306,20 @@ func TestRoutePrefix(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("Path "+opts.RoutePrefix+"/version with server ready test, Expected status 200 got: %s", resp.Status) } + + resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/snapshot", "", strings.NewReader("")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("Path %s/api/v2/admin/tsdb/snapshot with server unready test, Expected status 503 got: %s", opts.RoutePrefix, resp.Status) + } + + resp, err = http.Post("http://localhost:9091"+opts.RoutePrefix+"/api/v2/admin/tsdb/delete_series", "", strings.NewReader("{}")) + if err != nil { + t.Fatalf("Unexpected HTTP error %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("Path %s/api/v2/admin/tsdb/delete_series with server unready test, Expected status 503 got: %s", opts.RoutePrefix, resp.Status) + } }