github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #13607 from fionaliao/ooo-samples-appended-type

Browse Source 
Add sample type label to outOfOrderSamplesAppended metric
pull/13635/head
George Krajcsovits 9 months ago committed by GitHub
parent bf2658bec1 841a133514
commit c6c8f63516
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 43 additions and 33 deletions
Show Stats Download Patch File Download Diff File

14
tsdb/db_test.go
Unescape View File

@ -4945,7 +4945,7 @@ func Test_Querier_OOOQuery(t *testing.T) {
require.NotNil(t, seriesSet[series1.String()]) require.NotNil(t, seriesSet[series1.String()])
require.Len(t, seriesSet, 1) require.Len(t, seriesSet, 1)
require.Equal(t, expSamples, seriesSet[series1.String()]) require.Equal(t, expSamples, seriesSet[series1.String()])
require.GreaterOrEqual(t, float64(oooSamples), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch") requireEqualOOOSamples(t, oooSamples, db)
}) })
} }
} }
@ -5028,7 +5028,7 @@ func Test_ChunkQuerier_OOOQuery(t *testing.T) {
chks := queryChunks(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1")) chks := queryChunks(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1"))
require.NotNil(t, chks[series1.String()]) require.NotNil(t, chks[series1.String()])
require.Len(t, chks, 1) require.Len(t, chks, 1)
require.Equal(t, float64(oooSamples), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch") requireEqualOOOSamples(t, oooSamples, db)
var gotSamples []chunks.Sample var gotSamples []chunks.Sample
for _, chunk := range chks[series1.String()] { for _, chunk := range chks[series1.String()] {
it := chunk.Chunk.Iterator(nil) it := chunk.Chunk.Iterator(nil)
@ -5107,7 +5107,7 @@ func TestOOOAppendAndQuery(t *testing.T) {
} }
} }
require.Equal(t, expSamples, seriesSet) require.Equal(t, expSamples, seriesSet)
require.Equal(t, float64(totalSamples-2), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch") requireEqualOOOSamples(t, totalSamples-2, db)
} }
verifyOOOMinMaxTimes := func(expMin, expMax int64) { verifyOOOMinMaxTimes := func(expMin, expMax int64) {
@ -5216,7 +5216,7 @@ func TestOOODisabled(t *testing.T) {
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.")) seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar."))
require.Equal(t, expSamples, seriesSet) require.Equal(t, expSamples, seriesSet)
require.Equal(t, float64(0), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch") requireEqualOOOSamples(t, 0, db)
require.Equal(t, float64(failedSamples), require.Equal(t, float64(failedSamples),
prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))+prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))+prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)),
"number of ooo/oob samples mismatch") "number of ooo/oob samples mismatch")
@ -6943,3 +6943,9 @@ Outer:
require.NoError(t, writerErr) require.NoError(t, writerErr)
} }
func requireEqualOOOSamples(t *testing.T, expectedSamples int, db *DB) {
require.Equal(t, float64(expectedSamples),
prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat)),
"number of ooo appended samples mismatch")
}

6
tsdb/head.go
Unescape View File

@ -340,7 +340,7 @@ type headMetrics struct {
chunksRemoved prometheus.Counter chunksRemoved prometheus.Counter
gcDuration prometheus.Summary gcDuration prometheus.Summary
samplesAppended *prometheus.CounterVec samplesAppended *prometheus.CounterVec
outOfOrderSamplesAppended prometheus.Counter outOfOrderSamplesAppended *prometheus.CounterVec
outOfBoundSamples *prometheus.CounterVec outOfBoundSamples *prometheus.CounterVec
outOfOrderSamples *prometheus.CounterVec outOfOrderSamples *prometheus.CounterVec
tooOldSamples *prometheus.CounterVec tooOldSamples *prometheus.CounterVec
@ -420,10 +420,10 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
Name: "prometheus_tsdb_head_samples_appended_total", Name: "prometheus_tsdb_head_samples_appended_total",
Help: "Total number of appended samples.", Help: "Total number of appended samples.",
}, []string{"type"}), }, []string{"type"}),
outOfOrderSamplesAppended: prometheus.NewCounter(prometheus.CounterOpts{ outOfOrderSamplesAppended: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "prometheus_tsdb_head_out_of_order_samples_appended_total", Name: "prometheus_tsdb_head_out_of_order_samples_appended_total",
Help: "Total number of appended out of order samples.", Help: "Total number of appended out of order samples.",
}), }, []string{"type"}),
outOfBoundSamples: prometheus.NewCounterVec(prometheus.CounterOpts{ outOfBoundSamples: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "prometheus_tsdb_out_of_bound_samples_total", Name: "prometheus_tsdb_out_of_bound_samples_total",
Help: "Total number of out of bound samples ingestion failed attempts with out of order support disabled.", Help: "Total number of out of bound samples ingestion failed attempts with out of order support disabled.",

56
tsdb/head_append.go
Unescape View File

@ -835,11 +835,18 @@ func (a *headAppender) Commit() (err error) {
defer a.head.iso.closeAppend(a.appendID) defer a.head.iso.closeAppend(a.appendID)
var ( var (
samplesAppended = len(a.samples) floatsAppended = len(a.samples)
oooAccepted int // number of samples out of order but accepted: with ooo enabled and within time window histogramsAppended = len(a.histograms) + len(a.floatHistograms)
oooRejected int // number of samples rejected due to: out of order but OOO support disabled. // number of samples out of order but accepted: with ooo enabled and within time window
tooOldRejected int // number of samples rejected due to: that are out of order but too old (OOO support enabled, but outside time window) floatOOOAccepted int
oobRejected int // number of samples rejected due to: out of bounds: with t < minValidTime (OOO support disabled) // number of samples rejected due to: out of order but OOO support disabled.
floatOOORejected int
histoOOORejected int
// number of samples rejected due to: that are out of order but too old (OOO support enabled, but outside time window)
floatTooOldRejected int
// number of samples rejected due to: out of bounds: with t < minValidTime (OOO support disabled)
floatOOBRejected int
inOrderMint int64 = math.MaxInt64 inOrderMint int64 = math.MaxInt64
inOrderMaxt int64 = math.MinInt64 inOrderMaxt int64 = math.MinInt64
ooomint int64 = math.MaxInt64 ooomint int64 = math.MaxInt64
@ -902,16 +909,16 @@ func (a *headAppender) Commit() (err error) {
case err == nil: case err == nil:
// Do nothing. // Do nothing.
case errors.Is(err, storage.ErrOutOfOrderSample): case errors.Is(err, storage.ErrOutOfOrderSample):
samplesAppended-- floatsAppended--
oooRejected++ floatOOORejected++
case errors.Is(err, storage.ErrOutOfBounds): case errors.Is(err, storage.ErrOutOfBounds):
samplesAppended-- floatsAppended--
oobRejected++ floatOOBRejected++
case errors.Is(err, storage.ErrTooOldSample): case errors.Is(err, storage.ErrTooOldSample):
samplesAppended-- floatsAppended--
tooOldRejected++ floatTooOldRejected++
default: default:
samplesAppended-- floatsAppended--
} }
var ok, chunkCreated bool var ok, chunkCreated bool
@ -949,13 +956,13 @@ func (a *headAppender) Commit() (err error) {
if s.T > ooomaxt { if s.T > ooomaxt {
ooomaxt = s.T ooomaxt = s.T
} }
oooAccepted++ floatOOOAccepted++
} else { } else {
// Sample is an exact duplicate of the last sample. // Sample is an exact duplicate of the last sample.
// NOTE: We can only detect updates if they clash with a sample in the OOOHeadChunk, // NOTE: We can only detect updates if they clash with a sample in the OOOHeadChunk,
// not with samples in already flushed OOO chunks. // not with samples in already flushed OOO chunks.
// TODO(codesome): Add error reporting? It depends on addressing https://github.com/prometheus/prometheus/discussions/10305. // TODO(codesome): Add error reporting? It depends on addressing https://github.com/prometheus/prometheus/discussions/10305.
samplesAppended-- floatsAppended--
} }
default: default:
ok, chunkCreated = series.append(s.T, s.V, a.appendID, appendChunkOpts) ok, chunkCreated = series.append(s.T, s.V, a.appendID, appendChunkOpts)
@ -968,7 +975,7 @@ func (a *headAppender) Commit() (err error) {
} }
} else { } else {
// The sample is an exact duplicate, and should be silently dropped. // The sample is an exact duplicate, and should be silently dropped.
samplesAppended-- floatsAppended--
} }
} }
@ -982,8 +989,6 @@ func (a *headAppender) Commit() (err error) {
series.Unlock() series.Unlock()
} }
histogramsTotal := len(a.histograms)
histoOOORejected := 0
for i, s := range a.histograms { for i, s := range a.histograms {
series = a.histogramSeries[i] series = a.histogramSeries[i]
series.Lock() series.Lock()
@ -1000,7 +1005,7 @@ func (a *headAppender) Commit() (err error) {
inOrderMaxt = s.T inOrderMaxt = s.T
} }
} else { } else {
histogramsTotal-- histogramsAppended--
histoOOORejected++ histoOOORejected++
} }
if chunkCreated { if chunkCreated {
@ -1009,7 +1014,6 @@ func (a *headAppender) Commit() (err error) {
} }
} }
histogramsTotal += len(a.floatHistograms)
for i, s := range a.floatHistograms { for i, s := range a.floatHistograms {
series = a.floatHistogramSeries[i] series = a.floatHistogramSeries[i]
series.Lock() series.Lock()
@ -1026,7 +1030,7 @@ func (a *headAppender) Commit() (err error) {
inOrderMaxt = s.T inOrderMaxt = s.T
} }
} else { } else {
histogramsTotal-- histogramsAppended--
histoOOORejected++ histoOOORejected++
} }
if chunkCreated { if chunkCreated {
@ -1042,13 +1046,13 @@ func (a *headAppender) Commit() (err error) {
series.Unlock() series.Unlock()
} }
a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(oooRejected)) a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(floatOOORejected))
a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histoOOORejected)) a.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histoOOORejected))
a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(oobRejected)) a.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(floatOOBRejected))
a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(tooOldRejected)) a.head.metrics.tooOldSamples.WithLabelValues(sampleMetricTypeFloat).Add(float64(floatTooOldRejected))
a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(samplesAppended)) a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(floatsAppended))
a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histogramsTotal)) a.head.metrics.samplesAppended.WithLabelValues(sampleMetricTypeHistogram).Add(float64(histogramsAppended))
a.head.metrics.outOfOrderSamplesAppended.Add(float64(oooAccepted)) a.head.metrics.outOfOrderSamplesAppended.WithLabelValues(sampleMetricTypeFloat).Add(float64(floatOOOAccepted))
a.head.updateMinMaxTime(inOrderMint, inOrderMaxt) a.head.updateMinMaxTime(inOrderMint, inOrderMaxt)
a.head.updateMinOOOMaxOOOTime(ooomint, ooomaxt) a.head.updateMinOOOMaxOOOTime(ooomint, ooomaxt)

Write Preview
Loading…
Cancel
Save