github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update metrics for histograms 

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
pull/11395/head
Ganesh Vernekar 2022-10-03 13:23:28 +05:30
parent a01b3d7a14
commit d0a6488c74
No known key found for this signature in database
GPG Key ID: F056451B52F1DC34
3 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
tsdb/head.go
View File

@ -274,9 +274,9 @@ type headMetrics struct {
chunksCreated prometheus.Counter chunksCreated prometheus.Counter
chunksRemoved prometheus.Counter chunksRemoved prometheus.Counter
gcDuration prometheus.Summary gcDuration prometheus.Summary
samplesAppended prometheus.Counter samplesAppended *prometheus.CounterVec
outOfBoundSamples prometheus.Counter outOfBoundSamples *prometheus.CounterVec
outOfOrderSamples prometheus.Counter outOfOrderSamples *prometheus.CounterVec
walTruncateDuration prometheus.Summary walTruncateDuration prometheus.Summary
walCorruptionsTotal prometheus.Counter walCorruptionsTotal prometheus.Counter
walTotalReplayDuration prometheus.Gauge walTotalReplayDuration prometheus.Gauge
@ -288,12 +288,13 @@ type headMetrics struct {
checkpointCreationTotal prometheus.Counter checkpointCreationTotal prometheus.Counter
mmapChunkCorruptionTotal prometheus.Counter mmapChunkCorruptionTotal prometheus.Counter
snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1. snapshotReplayErrorTotal prometheus.Counter // Will be either 0 or 1.
// Sparse histogram metrics for experiments.
// TODO: remove these in the final version.
histogramSamplesTotal prometheus.Counter
} }
const (
sampleMetricTypeFloat = "float"
sampleMetricTypeHistogram = "histogram"
)
func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics { func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
m := &headMetrics{ m := &headMetrics{
activeAppenders: prometheus.NewGauge(prometheus.GaugeOpts{ activeAppenders: prometheus.NewGauge(prometheus.GaugeOpts{
@ -346,18 +347,18 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
Name: "prometheus_tsdb_data_replay_duration_seconds", Name: "prometheus_tsdb_data_replay_duration_seconds",
Help: "Time taken to replay the data on disk.", Help: "Time taken to replay the data on disk.",
}), }),
samplesAppended: prometheus.NewCounter(prometheus.CounterOpts{ samplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "prometheus_tsdb_head_samples_appended_total", Name: "prometheus_tsdb_head_samples_appended_total",
Help: "Total number of appended samples.", Help: "Total number of appended samples.",
}), }, []string{"type"}),
outOfBoundSamples: prometheus.NewCounter(prometheus.CounterOpts{ outOfBoundSamples: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "prometheus_tsdb_out_of_bound_samples_total", Name: "prometheus_tsdb_out_of_bound_samples_total",
Help: "Total number of out of bound samples ingestion failed attempts.", Help: "Total number of out of bound samples ingestion failed attempts.",
}), }, []string{"type"}),
outOfOrderSamples: prometheus.NewCounter(prometheus.CounterOpts{ outOfOrderSamples: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "prometheus_tsdb_out_of_order_samples_total", Name: "prometheus_tsdb_out_of_order_samples_total",
Help: "Total number of out of order samples ingestion failed attempts.", Help: "Total number of out of order samples ingestion failed attempts.",
}), }, []string{"type"}),
headTruncateFail: prometheus.NewCounter(prometheus.CounterOpts{ headTruncateFail: prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_tsdb_head_truncations_failed_total", Name: "prometheus_tsdb_head_truncations_failed_total",
Help: "Total number of head truncations that failed.", Help: "Total number of head truncations that failed.",
@ -390,10 +391,6 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
Name: "prometheus_tsdb_snapshot_replay_error_total", Name: "prometheus_tsdb_snapshot_replay_error_total",
Help: "Total number snapshot replays that failed.", Help: "Total number snapshot replays that failed.",
}), }),
histogramSamplesTotal: prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_tsdb_histogram_samples_total",
Help: "Total number of histograms samples added.",
}),
} }
if r != nil { if r != nil {
@ -421,7 +418,6 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
m.checkpointCreationTotal, m.checkpointCreationTotal,
m.mmapChunkCorruptionTotal, m.mmapChunkCorruptionTotal,
m.snapshotReplayErrorTotal, m.snapshotReplayErrorTotal,
m.histogramSamplesTotal,
// Metrics bound to functions and not needed in tests // Metrics bound to functions and not needed in tests
// can be created and registered on the spot. // can be created and registered on the spot.
prometheus.NewGaugeFunc(prometheus.GaugeOpts{ prometheus.NewGaugeFunc(prometheus.GaugeOpts{

23
tsdb/head_append.go
View File

@ -297,7 +297,7 @@ type headAppender struct {
func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) { func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
if t < a.minValidTime { if t < a.minValidTime {
a.head.metrics.outOfBoundSamples.Inc() a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Inc()
return 0, storage.ErrOutOfBounds return 0, storage.ErrOutOfBounds
} }
@ -335,7 +335,7 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64
if err := s.appendable(t, v); err != nil { if err := s.appendable(t, v); err != nil {
s.Unlock() s.Unlock()
if err == storage.ErrOutOfOrderSample { if err == storage.ErrOutOfOrderSample {
a.head.metrics.outOfOrderSamples.Inc() a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc()
} }
return 0, err return 0, err
} }
@ -444,7 +444,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
} }
if t < a.minValidTime { if t < a.minValidTime {
a.head.metrics.outOfBoundSamples.Inc() a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
return 0, storage.ErrOutOfBounds return 0, storage.ErrOutOfBounds
} }
@ -483,7 +483,7 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
if err := s.appendableHistogram(t, h); err != nil { if err := s.appendableHistogram(t, h); err != nil {
s.Unlock() s.Unlock()
if err == storage.ErrOutOfOrderSample { if err == storage.ErrOutOfOrderSample {
a.head.metrics.outOfOrderSamples.Inc() a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
} }
return 0, err return 0, err
} }
@ -728,7 +728,7 @@ func (a *headAppender) Commit() (err error) {
if !ok { if !ok {
total-- total--
a.head.metrics.outOfOrderSamples.Inc() a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Inc()
} }
if chunkCreated { if chunkCreated {
a.head.metrics.chunks.Inc() a.head.metrics.chunks.Inc()
@ -736,7 +736,7 @@ func (a *headAppender) Commit() (err error) {
} }
} }
total += len(a.histograms) // TODO: different metric? histogramsTotal := len(a.histograms)
for i, s := range a.histograms { for i, s := range a.histograms {
series = a.histogramSeries[i] series = a.histogramSeries[i]
series.Lock() series.Lock()
@ -745,11 +745,9 @@ func (a *headAppender) Commit() (err error) {
series.pendingCommit = false series.pendingCommit = false
series.Unlock() series.Unlock()
if ok { if !ok {
a.head.metrics.histogramSamplesTotal.Inc() histogramsTotal--
} else { a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Inc()
total--
a.head.metrics.outOfOrderSamples.Inc()
} }
if chunkCreated { if chunkCreated {
a.head.metrics.chunks.Inc() a.head.metrics.chunks.Inc()
@ -764,7 +762,8 @@ func (a *headAppender) Commit() (err error) {
series.Unlock() series.Unlock()
} }
a.head.metrics.samplesAppended.Add(float64(total)) a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(total))
a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histogramsTotal))
a.head.updateMinMaxTime(a.mint, a.maxt) a.head.updateMinMaxTime(a.mint, a.maxt)
return nil return nil

22
tsdb/head_test.go
View File

@ -2131,19 +2131,19 @@ func TestOutOfOrderSamplesMetric(t *testing.T) {
require.NoError(t, app.Commit()) require.NoError(t, app.Commit())
// Test out of order metric. // Test out of order metric.
require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
app = db.Appender(ctx) app = db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), 2, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), 2, 99)
require.Equal(t, storage.ErrOutOfOrderSample, err) require.Equal(t, storage.ErrOutOfOrderSample, err)
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
_, err = app.Append(0, labels.FromStrings("a", "b"), 3, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), 3, 99)
require.Equal(t, storage.ErrOutOfOrderSample, err) require.Equal(t, storage.ErrOutOfOrderSample, err)
require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
_, err = app.Append(0, labels.FromStrings("a", "b"), 4, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), 4, 99)
require.Equal(t, storage.ErrOutOfOrderSample, err) require.Equal(t, storage.ErrOutOfOrderSample, err)
require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) require.Equal(t, 3.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
require.NoError(t, app.Commit()) require.NoError(t, app.Commit())
// Compact Head to test out of bound metric. // Compact Head to test out of bound metric.
@ -2159,11 +2159,11 @@ func TestOutOfOrderSamplesMetric(t *testing.T) {
app = db.Appender(ctx) app = db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-2, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-2, 99)
require.Equal(t, storage.ErrOutOfBounds, err) require.Equal(t, storage.ErrOutOfBounds, err)
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples)) require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)))
_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-1, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()-1, 99)
require.Equal(t, storage.ErrOutOfBounds, err) require.Equal(t, storage.ErrOutOfBounds, err)
require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples)) require.Equal(t, 2.0, prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)))
require.NoError(t, app.Commit()) require.NoError(t, app.Commit())
// Some more valid samples for out of order. // Some more valid samples for out of order.
@ -2178,15 +2178,15 @@ func TestOutOfOrderSamplesMetric(t *testing.T) {
app = db.Appender(ctx) app = db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+2, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+2, 99)
require.Equal(t, storage.ErrOutOfOrderSample, err) require.Equal(t, storage.ErrOutOfOrderSample, err)
require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) require.Equal(t, 4.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+3, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+3, 99)
require.Equal(t, storage.ErrOutOfOrderSample, err) require.Equal(t, storage.ErrOutOfOrderSample, err)
require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) require.Equal(t, 5.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
_, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+4, 99) _, err = app.Append(0, labels.FromStrings("a", "b"), db.head.minValidTime.Load()+DefaultBlockDuration+4, 99)
require.Equal(t, storage.ErrOutOfOrderSample, err) require.Equal(t, storage.ErrOutOfOrderSample, err)
require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples)) require.Equal(t, 6.0, prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat)))
require.NoError(t, app.Commit()) require.NoError(t, app.Commit())
} }
@ -3267,7 +3267,7 @@ func TestHistogramMetrics(t *testing.T) {
} }
} }
require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.histogramSamplesTotal)) require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram)))
require.NoError(t, head.Close()) require.NoError(t, head.Close())
w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false) w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false)
@ -3276,7 +3276,7 @@ func TestHistogramMetrics(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, head.Init(0)) require.NoError(t, head.Init(0))
require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.histogramSamplesTotal)) // Counter reset. require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram))) // Counter reset.
} }
func TestHistogramStaleSample(t *testing.T) { func TestHistogramStaleSample(t *testing.T) {