Merge pull request #13681 from krajorama/native-latency-histograms

Add native histograms to latency/duration metrics

documentation/examples/remote_storage/remote_storage_adapter/main.go

@@ -83,9 +83,12 @@ var (
 	)
 	sentBatchDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name:    "sent_batch_duration_seconds",
+			Name:                            "sent_batch_duration_seconds",
-			Help:    "Duration of sample batch send calls to the remote storage.",
+			Help:                            "Duration of sample batch send calls to the remote storage.",
-			Buckets: prometheus.DefBuckets,
+			Buckets:                         prometheus.DefBuckets,
+			NativeHistogramBucketFactor:     1.1,
+			NativeHistogramMaxBucketNumber:  100,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
 		},
 		[]string{"remote"},
 	)

storage/remote/client.go

@@ -64,11 +64,14 @@ var (
 	)
 	remoteReadQueryDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Namespace: namespace,
+			Namespace:                       namespace,
-			Subsystem: subsystem,
+			Subsystem:                       subsystem,
-			Name:      "read_request_duration_seconds",
+			Name:                            "read_request_duration_seconds",
-			Help:      "Histogram of the latency for remote read requests.",
+			Help:                            "Histogram of the latency for remote read requests.",
-			Buckets:   append(prometheus.DefBuckets, 25, 60),
+			Buckets:                         append(prometheus.DefBuckets, 25, 60),
+			NativeHistogramBucketFactor:     1.1,
+			NativeHistogramMaxBucketNumber:  100,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
 		},
 		[]string{remoteName, endpoint},
 	)

storage/remote/queue_manager.go

@@ -214,12 +214,15 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager
 		ConstLabels: constLabels,
 	})
 	m.sentBatchDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
-		Namespace:   namespace,
+		Namespace:                       namespace,
-		Subsystem:   subsystem,
+		Subsystem:                       subsystem,
-		Name:        "sent_batch_duration_seconds",
+		Name:                            "sent_batch_duration_seconds",
-		Help:        "Duration of send calls to the remote storage.",
+		Help:                            "Duration of send calls to the remote storage.",
-		Buckets:     append(prometheus.DefBuckets, 25, 60, 120, 300),
+		Buckets:                         append(prometheus.DefBuckets, 25, 60, 120, 300),
-		ConstLabels: constLabels,
+		ConstLabels:                     constLabels,
+		NativeHistogramBucketFactor:     1.1,
+		NativeHistogramMaxBucketNumber:  100,
+		NativeHistogramMinResetDuration: 1 * time.Hour,
 	})
 	m.highestSentTimestamp = &maxTimestamp{
 		Gauge: prometheus.NewGauge(prometheus.GaugeOpts{

tsdb/compact.go

@@ -112,9 +112,12 @@ func newCompactorMetrics(r prometheus.Registerer) *CompactorMetrics {
 		Help: "Total number of compactions done on overlapping blocks.",
 	})
 	m.Duration = prometheus.NewHistogram(prometheus.HistogramOpts{
-		Name:    "prometheus_tsdb_compaction_duration_seconds",
+		Name:                            "prometheus_tsdb_compaction_duration_seconds",
-		Help:    "Duration of compaction runs",
+		Help:                            "Duration of compaction runs",
-		Buckets: prometheus.ExponentialBuckets(1, 2, 14),
+		Buckets:                         prometheus.ExponentialBuckets(1, 2, 14),
+		NativeHistogramBucketFactor:     1.1,
+		NativeHistogramMaxBucketNumber:  100,
+		NativeHistogramMinResetDuration: 1 * time.Hour,
 	})
 	m.ChunkSize = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name:    "prometheus_tsdb_compaction_chunk_size_bytes",

tsdb/db.go

@@ -319,8 +319,11 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics {
 		return float64(db.blocks[0].meta.MinTime)
 	})
 	m.tombCleanTimer = prometheus.NewHistogram(prometheus.HistogramOpts{
-		Name: "prometheus_tsdb_tombstone_cleanup_seconds",
+		Name:                            "prometheus_tsdb_tombstone_cleanup_seconds",
-		Help: "The time taken to recompact blocks to remove tombstones.",
+		Help:                            "The time taken to recompact blocks to remove tombstones.",
+		NativeHistogramBucketFactor:     1.1,
+		NativeHistogramMaxBucketNumber:  100,
+		NativeHistogramMinResetDuration: 1 * time.Hour,
 	})
 	m.blocksBytes = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "prometheus_tsdb_storage_blocks_bytes",

tsdb/head.go

@@ -480,6 +480,9 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
 				60 * 60 * 6,  // 6h
 				60 * 60 * 12, // 12h
 			},
+			NativeHistogramBucketFactor:     1.1,
+			NativeHistogramMaxBucketNumber:  100,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
 		}),
 		mmapChunksTotal: prometheus.NewCounter(prometheus.CounterOpts{
 			Name: "prometheus_tsdb_mmap_chunks_total",

web/web.go

@@ -123,9 +123,12 @@ func newMetrics(r prometheus.Registerer) *metrics {
 		),
 		requestDuration: prometheus.NewHistogramVec(
 			prometheus.HistogramOpts{
-				Name:    "prometheus_http_request_duration_seconds",
+				Name:                            "prometheus_http_request_duration_seconds",
-				Help:    "Histogram of latencies for HTTP requests.",
+				Help:                            "Histogram of latencies for HTTP requests.",
-				Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
+				Buckets:                         []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
+				NativeHistogramBucketFactor:     1.1,
+				NativeHistogramMaxBucketNumber:  100,
+				NativeHistogramMinResetDuration: 1 * time.Hour,
 			},
 			[]string{"handler"},
 		),