From ffefc8e74dd59143f93af4cecbbae4799bce8c35 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Tue, 20 Nov 2018 18:11:40 +0100 Subject: [PATCH] Add a limit to the number of in-flight requests (#1166) In order to avoid stuck collectors using up all system resources, add a limit to the number of parallel in-flight scrape requests. This will return a 503 error. Default to 40 requests, this seems like a reasonable number based on: * Two Prometheus servers scraping every 15 seconds. * Failing scrapes after 5 minutes of stuckness. Signed-off-by: Ben Kochie --- node_exporter.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/node_exporter.go b/node_exporter.go index 0475a6aa..550d7b13 100644 --- a/node_exporter.go +++ b/node_exporter.go @@ -36,12 +36,14 @@ type handler struct { // the exporter itself. exporterMetricsRegistry *prometheus.Registry includeExporterMetrics bool + maxRequests int } -func newHandler(includeExporterMetrics bool) *handler { +func newHandler(includeExporterMetrics bool, maxRequests int) *handler { h := &handler{ exporterMetricsRegistry: prometheus.NewRegistry(), includeExporterMetrics: includeExporterMetrics, + maxRequests: maxRequests, } if h.includeExporterMetrics { h.exporterMetricsRegistry.MustRegister( @@ -111,8 +113,9 @@ func (h *handler) innerHandler(filters ...string) (http.Handler, error) { handler := promhttp.HandlerFor( prometheus.Gatherers{h.exporterMetricsRegistry, r}, promhttp.HandlerOpts{ - ErrorLog: log.NewErrorLogger(), - ErrorHandling: promhttp.ContinueOnError, + ErrorLog: log.NewErrorLogger(), + ErrorHandling: promhttp.ContinueOnError, + MaxRequestsInFlight: h.maxRequests, }, ) if h.includeExporterMetrics { @@ -139,6 +142,10 @@ func main() { "web.disable-exporter-metrics", "Exclude metrics about the exporter itself (promhttp_*, process_*, go_*).", ).Bool() + maxRequests = kingpin.Flag( + "web.max-requests", + "Maximum number of parallel scrape requests. Use 0 to disable.", + ).Default("40").Int() ) log.AddFlags(kingpin.CommandLine) @@ -149,7 +156,7 @@ func main() { log.Infoln("Starting node_exporter", version.Info()) log.Infoln("Build context", version.BuildContext()) - http.Handle(*metricsPath, newHandler(!*disableExporterMetrics)) + http.Handle(*metricsPath, newHandler(!*disableExporterMetrics, *maxRequests)) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(` Node Exporter