Move prometheus_ready metric to web package (#10729)

This moves prometheus_ready to the web package and links it with the ready variable that decides if HTTP requests should return 200 or 503.
This is a follow up change from #10682

Signed-off-by: Łukasz Mierzwa <l.mierzwa@gmail.com>

cmd/prometheus/main.go

@@ -86,10 +86,6 @@ var (
 		Name: "prometheus_config_last_reload_success_timestamp_seconds",
 		Help: "Timestamp of the last successful configuration reload.",
 	})
-	readyStatus = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "prometheus_ready",
-		Help: "Whether Prometheus startup was fully completed and the server is ready for normal operation.",
-	})
 
 	defaultRetentionString   = "15d"
 	defaultRetentionDuration model.Duration
@@ -756,7 +752,6 @@ func main() {
 
 	prometheus.MustRegister(configSuccess)
 	prometheus.MustRegister(configSuccessTime)
-	prometheus.MustRegister(readyStatus)
 
 	// Start all components while we wait for TSDB to open but only load
 	// initial config and mark ourselves as ready after it completed.
@@ -812,6 +807,7 @@ func main() {
 			},
 			func(err error) {
 				close(cancel)
+				webHandler.SetReady(false)
 			},
 		)
 	}
@@ -949,9 +945,8 @@ func main() {
 
 				reloadReady.Close()
 
-				webHandler.Ready()
+				webHandler.SetReady(true)
 				level.Info(logger).Log("msg", "Server is ready to receive web requests.")
-				readyStatus.Set(1)
 				<-cancel
 				return nil
 			},

web/web.go

@@ -109,6 +109,7 @@ type metrics struct {
 	requestCounter  *prometheus.CounterVec
 	requestDuration *prometheus.HistogramVec
 	responseSize    *prometheus.HistogramVec
+	readyStatus     prometheus.Gauge
 }
 
 func newMetrics(r prometheus.Registerer) *metrics {
@@ -136,10 +137,14 @@ func newMetrics(r prometheus.Registerer) *metrics {
 			},
 			[]string{"handler"},
 		),
+		readyStatus: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "prometheus_ready",
+			Help: "Whether Prometheus startup was fully completed and the server is ready for normal operation.",
+		}),
 	}
 
 	if r != nil {
-		r.MustRegister(m.requestCounter, m.requestDuration, m.responseSize)
+		r.MustRegister(m.requestCounter, m.requestDuration, m.responseSize, m.readyStatus)
 		registerFederationMetrics(r)
 	}
 	return m
@@ -301,7 +306,7 @@ func New(logger log.Logger, o *Options) *Handler {
 
 		now: model.Now,
 	}
-	h.ready.Store(0)
+	h.SetReady(false)
 
 	factoryTr := func(_ context.Context) api_v1.TargetRetriever { return h.scrapeManager }
 	factoryAr := func(_ context.Context) api_v1.AlertmanagerRetriever { return h.notifier }
@@ -503,9 +508,16 @@ func serveDebug(w http.ResponseWriter, req *http.Request) {
 	}
 }
 
-// Ready sets Handler to be ready.
+// SetReady sets the ready status of our web Handler
-func (h *Handler) Ready() {
+func (h *Handler) SetReady(v bool) {
-	h.ready.Store(1)
+	if v {
+		h.ready.Store(1)
+		h.metrics.readyStatus.Set(1)
+		return
+	}
+
+	h.ready.Store(0)
+	h.metrics.readyStatus.Set(0)
 }
 
 // Verifies whether the server is ready or not.

web/web_test.go

@@ -146,7 +146,7 @@ func TestReadyAndHealthy(t *testing.T) {
 	cleanupTestResponse(t, resp)
 
 	// Set to ready.
-	webHandler.Ready()
+	webHandler.SetReady(true)
 
 	for _, u := range []string{
 		baseURL + "/-/healthy",
@@ -245,7 +245,7 @@ func TestRoutePrefix(t *testing.T) {
 	cleanupTestResponse(t, resp)
 
 	// Set to ready.
-	webHandler.Ready()
+	webHandler.SetReady(true)
 
 	resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/healthy")
 	require.NoError(t, err)
@@ -292,7 +292,7 @@ func TestDebugHandler(t *testing.T) {
 			},
 		}
 		handler := New(nil, opts)
-		handler.Ready()
+		handler.SetReady(true)
 
 		w := httptest.NewRecorder()
 
@@ -329,16 +329,28 @@ func TestHTTPMetrics(t *testing.T) {
 
 	code := getReady()
 	require.Equal(t, http.StatusServiceUnavailable, code)
+	ready := handler.metrics.readyStatus
+	require.Equal(t, 0, int(prom_testutil.ToFloat64(ready)))
 	counter := handler.metrics.requestCounter
 	require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
 
-	handler.Ready()
+	handler.SetReady(true)
 	for range [2]int{} {
 		code = getReady()
 		require.Equal(t, http.StatusOK, code)
 	}
+	require.Equal(t, 1, int(prom_testutil.ToFloat64(ready)))
 	require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
 	require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
+
+	handler.SetReady(false)
+	for range [2]int{} {
+		code = getReady()
+		require.Equal(t, http.StatusServiceUnavailable, code)
+	}
+	require.Equal(t, 0, int(prom_testutil.ToFloat64(ready)))
+	require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
+	require.Equal(t, 3, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
 }
 
 func TestShutdownWithStaleConnection(t *testing.T) {
@@ -510,7 +522,7 @@ func TestAgentAPIEndPoints(t *testing.T) {
 	opts.Flags = map[string]string{}
 
 	webHandler := New(nil, opts)
-	webHandler.Ready()
+	webHandler.SetReady(true)
 	webHandler.config = &config.Config{}
 	webHandler.notifier = &notifier.Manager{}
 	l, err := webHandler.Listener()