@ -321,8 +321,8 @@ type headAppender struct {
}
func ( a * headAppender ) Append ( ref storage . SeriesRef , lset labels . Labels , t int64 , v float64 ) ( storage . SeriesRef , error ) {
// F or OOO inserts, this restriction is irrelevant and will be checked later once we confirm the sample is an in-order append .
// If OOO inserts are disabled, we may as well as check this as early as we can and avoid more work .
// F ail fast if OOO is disabled and the sample is out of bounds .
// Otherwise a full check will be done later to decide if the sample is in-order or out-of-order .
if a . oooTimeWindow == 0 && t < a . minValidTime {
a . head . metrics . outOfBoundSamples . WithLabelValues ( sampleMetricTypeFloat ) . Inc ( )
return 0 , storage . ErrOutOfBounds
@ -493,46 +493,94 @@ func (s *memSeries) appendable(t int64, v float64, headMaxt, minValidTime, oooTi
return false , headMaxt - t , storage . ErrOutOfOrderSample
}
// appendableHistogram checks whether the given histogram is valid for appending to the series.
func ( s * memSeries ) appendableHistogram ( t int64 , h * histogram . Histogram ) error {
if s . headChunks == nil {
return nil
// appendableHistogram checks whether the given histogram sample is valid for appending to the series. (if we return false and no error)
// The sample belongs to the out of order chunk if we return true and no error.
// An error signifies the sample cannot be handled.
func ( s * memSeries ) appendableHistogram ( t int64 , h * histogram . Histogram , headMaxt , minValidTime , oooTimeWindow int64 , oooHistogramsEnabled bool ) ( isOOO bool , oooDelta int64 , err error ) {
// Check if we can append in the in-order chunk.
if t >= minValidTime {
if s . headChunks == nil {
// The series has no sample and was freshly created.
return false , 0 , nil
}
msMaxt := s . maxTime ( )
if t > msMaxt {
return false , 0 , nil
}
if t == msMaxt {
// We are allowing exact duplicates as we can encounter them in valid cases
// like federation and erroring out at that time would be extremely noisy.
// This only checks against the latest in-order sample.
// The OOO headchunk has its own method to detect these duplicates.
if ! h . Equals ( s . lastHistogramValue ) {
return false , 0 , storage . ErrDuplicateSampleForTimestamp
}
// Sample is identical (ts + value) with most current (highest ts) sample in sampleBuf.
return false , 0 , nil
}
}
if t > s . headChunks . maxTime {
return nil
}
if t < s . headChunks . maxTime {
return storage . ErrOutOfOrderSample
// The sample cannot go in the in-order chunk. Check if it can go in the out-of-order chunk.
if oooTimeWindow > 0 && t >= headMaxt - oooTimeWindow {
if ! oooHistogramsEnabled {
return true , headMaxt - t , storage . ErrOOONativeHistogramsDisabled
}
return true , headMaxt - t , nil
}
// We are allowing exact duplicates as we can encounter them in valid cases
// like federation and erroring out at that time would be extremely noisy.
if ! h . Equals ( s . lastHistogramValue ) {
return storage . ErrDuplicateSampleForTimestamp
// The sample cannot go in both in-order and out-of-order chunk.
if oooTimeWindow > 0 {
return true , headMaxt - t , storage . ErrTooOldSample
}
return nil
if t < minValidTime {
return false , headMaxt - t , storage . ErrOutOfBounds
}
return false , headMaxt - t , storage . ErrOutOfOrderSample
}
// appendableFloatHistogram checks whether the given float histogram is valid for appending to the series.
func ( s * memSeries ) appendableFloatHistogram ( t int64 , fh * histogram . FloatHistogram ) error {
if s . headChunks == nil {
return nil
// appendableFloatHistogram checks whether the given float histogram sample is valid for appending to the series. (if we return false and no error)
// The sample belongs to the out of order chunk if we return true and no error.
// An error signifies the sample cannot be handled.
func ( s * memSeries ) appendableFloatHistogram ( t int64 , fh * histogram . FloatHistogram , headMaxt , minValidTime , oooTimeWindow int64 , oooHistogramsEnabled bool ) ( isOOO bool , oooDelta int64 , err error ) {
// Check if we can append in the in-order chunk.
if t >= minValidTime {
if s . headChunks == nil {
// The series has no sample and was freshly created.
return false , 0 , nil
}
msMaxt := s . maxTime ( )
if t > msMaxt {
return false , 0 , nil
}
if t == msMaxt {
// We are allowing exact duplicates as we can encounter them in valid cases
// like federation and erroring out at that time would be extremely noisy.
// This only checks against the latest in-order sample.
// The OOO headchunk has its own method to detect these duplicates.
if ! fh . Equals ( s . lastFloatHistogramValue ) {
return false , 0 , storage . ErrDuplicateSampleForTimestamp
}
// Sample is identical (ts + value) with most current (highest ts) sample in sampleBuf.
return false , 0 , nil
}
}
if t > s . headChunks . maxTime {
return nil
}
if t < s . headChunks . maxTime {
return storage . ErrOutOfOrderSample
// The sample cannot go in the in-order chunk. Check if it can go in the out-of-order chunk.
if oooTimeWindow > 0 && t >= headMaxt - oooTimeWindow {
if ! oooHistogramsEnabled {
return true , headMaxt - t , storage . ErrOOONativeHistogramsDisabled
}
return true , headMaxt - t , nil
}
// We are allowing exact duplicates as we can encounter them in valid cases
// like federation and erroring out at that time would be extremely noisy.
if ! fh . Equals ( s . lastFloatHistogramValue ) {
return storage . ErrDuplicateSampleForTimestamp
// The sample cannot go in both in-order and out-of-order chunk.
if oooTimeWindow > 0 {
return true , headMaxt - t , storage . ErrTooOldSample
}
return nil
if t < minValidTime {
return false , headMaxt - t , storage . ErrOutOfBounds
}
return false , headMaxt - t , storage . ErrOutOfOrderSample
}
// AppendExemplar for headAppender assumes the series ref already exists, and so it doesn't
@ -577,7 +625,9 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
return 0 , storage . ErrNativeHistogramsDisabled
}
if t < a . minValidTime {
// Fail fast if OOO is disabled and the sample is out of bounds.
// Otherwise a full check will be done later to decide if the sample is in-order or out-of-order.
if ( a . oooTimeWindow == 0 || ! a . head . opts . EnableOOONativeHistograms . Load ( ) ) && t < a . minValidTime {
a . head . metrics . outOfBoundSamples . WithLabelValues ( sampleMetricTypeHistogram ) . Inc ( )
return 0 , storage . ErrOutOfBounds
}
@ -629,15 +679,27 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
switch {
case h != nil :
s . Lock ( )
if err := s . appendableHistogram ( t , h ) ; err != nil {
s . Unlock ( )
if errors . Is ( err , storage . ErrOutOfOrderSample ) {
// TODO(codesome): If we definitely know at this point that the sample is ooo, then optimise
// to skip that sample from the WAL and write only in the WBL.
_ , delta , err := s . appendableHistogram ( t , h , a . headMaxt , a . minValidTime , a . oooTimeWindow , a . head . opts . EnableOOONativeHistograms . Load ( ) )
if err != nil {
s . pendingCommit = true
}
s . Unlock ( )
if delta > 0 {
a . head . metrics . oooHistogram . Observe ( float64 ( delta ) / 1000 )
}
if err != nil {
switch {
case errors . Is ( err , storage . ErrOutOfOrderSample ) :
fallthrough
case errors . Is ( err , storage . ErrOOONativeHistogramsDisabled ) :
a . head . metrics . outOfOrderSamples . WithLabelValues ( sampleMetricTypeHistogram ) . Inc ( )
case errors . Is ( err , storage . ErrTooOldSample ) :
a . head . metrics . tooOldSamples . WithLabelValues ( sampleMetricTypeHistogram ) . Inc ( )
}
return 0 , err
}
s . pendingCommit = true
s . Unlock ( )
a . histograms = append ( a . histograms , record . RefHistogramSample {
Ref : s . ref ,
T : t ,
@ -646,15 +708,27 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels
a . histogramSeries = append ( a . histogramSeries , s )
case fh != nil :
s . Lock ( )
if err := s . appendableFloatHistogram ( t , fh ) ; err != nil {
s . Unlock ( )
if errors . Is ( err , storage . ErrOutOfOrderSample ) {
// TODO(codesome): If we definitely know at this point that the sample is ooo, then optimise
// to skip that sample from the WAL and write only in the WBL.
_ , delta , err := s . appendableFloatHistogram ( t , fh , a . headMaxt , a . minValidTime , a . oooTimeWindow , a . head . opts . EnableOOONativeHistograms . Load ( ) )
if err == nil {
s . pendingCommit = true
}
s . Unlock ( )
if delta > 0 {
a . head . metrics . oooHistogram . Observe ( float64 ( delta ) / 1000 )
}
if err != nil {
switch {
case errors . Is ( err , storage . ErrOutOfOrderSample ) :
fallthrough
case errors . Is ( err , storage . ErrOOONativeHistogramsDisabled ) :
a . head . metrics . outOfOrderSamples . WithLabelValues ( sampleMetricTypeHistogram ) . Inc ( )
case errors . Is ( err , storage . ErrTooOldSample ) :
a . head . metrics . tooOldSamples . WithLabelValues ( sampleMetricTypeHistogram ) . Inc ( )
}
return 0 , err
}
s . pendingCommit = true
s . Unlock ( )
a . floatHistograms = append ( a . floatHistograms , record . RefFloatHistogramSample {
Ref : s . ref ,
T : t ,
@ -841,20 +915,24 @@ func (a *headAppender) Commit() (err error) {
floatsAppended = len ( a . samples )
histogramsAppended = len ( a . histograms ) + len ( a . floatHistograms )
// number of samples out of order but accepted: with ooo enabled and within time window
oooFloatsAccepted int
oooFloatsAccepted int
oooHistogramAccepted int
// number of samples rejected due to: out of order but OOO support disabled.
floatOOORejected int
histoOOORejected int
// number of samples rejected due to: that are out of order but too old (OOO support enabled, but outside time window)
floatTooOldRejected int
histoTooOldRejected int
// number of samples rejected due to: out of bounds: with t < minValidTime (OOO support disabled)
floatOOBRejected int
floatOOBRejected int
histoOOBRejected int
inOrderMint int64 = math . MaxInt64
inOrderMaxt int64 = math . MinInt64
oooMinT int64 = math . MaxInt64
oooMaxT int64 = math . MinInt64
wblSamples [ ] record . RefSample
wblHistograms [ ] record . RefHistogramSample
wblFloatHistograms [ ] record . RefFloatHistogramSample
oooMmapMarkers map [ chunks . HeadSeriesRef ] [ ] chunks . ChunkDiskMapperRef
oooMmapMarkersCount int
oooRecords [ ] [ ] byte
@ -876,6 +954,8 @@ func (a *headAppender) Commit() (err error) {
if a . head . wbl == nil {
// WBL is not enabled. So no need to collect.
wblSamples = nil
wblHistograms = nil
wblFloatHistograms = nil
oooMmapMarkers = nil
oooMmapMarkersCount = 0
return
@ -903,8 +983,18 @@ func (a *headAppender) Commit() (err error) {
r := enc . Samples ( wblSamples , a . head . getBytesBuffer ( ) )
oooRecords = append ( oooRecords , r )
}
if len ( wblHistograms ) > 0 {
r := enc . HistogramSamples ( wblHistograms , a . head . getBytesBuffer ( ) )
oooRecords = append ( oooRecords , r )
}
if len ( wblFloatHistograms ) > 0 {
r := enc . FloatHistogramSamples ( wblFloatHistograms , a . head . getBytesBuffer ( ) )
oooRecords = append ( oooRecords , r )
}
wblSamples = nil
wblHistograms = nil
wblFloatHistograms = nil
oooMmapMarkers = nil
}
for i , s := range a . samples {
@ -1006,51 +1096,193 @@ func (a *headAppender) Commit() (err error) {
for i , s := range a . histograms {
series = a . histogramSeries [ i ]
series . Lock ( )
ok , chunkCreated := series . appendHistogram ( s . T , s . H , a . appendID , appendChunkOpts )
series . cleanupAppendIDsBelow ( a . cleanupAppendIDsBelow )
series . pendingCommit = false
series . Unlock ( )
if ok {
if s . T < inOrderMint {
inOrderMint = s . T
}
if s . T > inOrderMaxt {
inOrderMaxt = s . T
}
} else {
oooSample , _ , err := series . appendableHistogram ( s . T , s . H , a . headMaxt , a . minValidTime , a . oooTimeWindow , a . head . opts . EnableOOONativeHistograms . Load ( ) )
switch {
case err == nil :
// Do nothing.
case errors . Is ( err , storage . ErrOutOfOrderSample ) :
histogramsAppended --
histoOOORejected ++
case errors . Is ( err , storage . ErrOutOfBounds ) :
histogramsAppended --
histoOOBRejected ++
case errors . Is ( err , storage . ErrTooOldSample ) :
histogramsAppended --
histoTooOldRejected ++
default :
histogramsAppended --
}
var ok , chunkCreated bool
switch {
case err != nil :
// Do nothing here.
case oooSample :
// Sample is OOO and OOO handling is enabled
// and the delta is within the OOO tolerance.
var mmapRefs [ ] chunks . ChunkDiskMapperRef
ok , chunkCreated , mmapRefs = series . insert ( s . T , 0 , s . H , nil , a . head . chunkDiskMapper , oooCapMax , a . head . logger )
if chunkCreated {
r , ok := oooMmapMarkers [ series . ref ]
if ! ok || r != nil {
// !ok means there are no markers collected for these samples yet. So we first flush the samples
// before setting this m-map marker.
// r != 0 means we have already m-mapped a chunk for this series in the same Commit().
// Hence, before we m-map again, we should add the samples and m-map markers
// seen till now to the WBL records.
collectOOORecords ( )
}
if oooMmapMarkers == nil {
oooMmapMarkers = make ( map [ chunks . HeadSeriesRef ] [ ] chunks . ChunkDiskMapperRef )
}
if len ( mmapRefs ) > 0 {
oooMmapMarkers [ series . ref ] = mmapRefs
oooMmapMarkersCount += len ( mmapRefs )
} else {
// No chunk was written to disk, so we need to set an initial marker for this series.
oooMmapMarkers [ series . ref ] = [ ] chunks . ChunkDiskMapperRef { 0 }
oooMmapMarkersCount ++
}
}
if ok {
wblHistograms = append ( wblHistograms , s )
if s . T < oooMinT {
oooMinT = s . T
}
if s . T > oooMaxT {
oooMaxT = s . T
}
oooHistogramAccepted ++
} else {
// Sample is an exact duplicate of the last sample.
// NOTE: We can only detect updates if they clash with a sample in the OOOHeadChunk,
// not with samples in already flushed OOO chunks.
// TODO(codesome): Add error reporting? It depends on addressing https://github.com/prometheus/prometheus/discussions/10305.
histogramsAppended --
}
default :
ok , chunkCreated = series . appendHistogram ( s . T , s . H , a . appendID , appendChunkOpts )
if ok {
if s . T < inOrderMint {
inOrderMint = s . T
}
if s . T > inOrderMaxt {
inOrderMaxt = s . T
}
} else {
histogramsAppended --
histoOOORejected ++
}
}
if chunkCreated {
a . head . metrics . chunks . Inc ( )
a . head . metrics . chunksCreated . Inc ( )
}
series . cleanupAppendIDsBelow ( a . cleanupAppendIDsBelow )
series . pendingCommit = false
series . Unlock ( )
}
for i , s := range a . floatHistograms {
series = a . floatHistogramSeries [ i ]
series . Lock ( )
ok , chunkCreated := series . appendFloatHistogram ( s . T , s . FH , a . appendID , appendChunkOpts )
series . cleanupAppendIDsBelow ( a . cleanupAppendIDsBelow )
series . pendingCommit = false
series . Unlock ( )
if ok {
if s . T < inOrderMint {
inOrderMint = s . T
}
if s . T > inOrderMaxt {
inOrderMaxt = s . T
}
} else {
oooSample , _ , err := series . appendableFloatHistogram ( s . T , s . FH , a . headMaxt , a . minValidTime , a . oooTimeWindow , a . head . opts . EnableOOONativeHistograms . Load ( ) )
switch {
case err == nil :
// Do nothing.
case errors . Is ( err , storage . ErrOutOfOrderSample ) :
histogramsAppended --
histoOOORejected ++
case errors . Is ( err , storage . ErrOutOfBounds ) :
histogramsAppended --
histoOOBRejected ++
case errors . Is ( err , storage . ErrTooOldSample ) :
histogramsAppended --
histoTooOldRejected ++
default :
histogramsAppended --
}
var ok , chunkCreated bool
switch {
case err != nil :
// Do nothing here.
case oooSample :
// Sample is OOO and OOO handling is enabled
// and the delta is within the OOO tolerance.
var mmapRefs [ ] chunks . ChunkDiskMapperRef
ok , chunkCreated , mmapRefs = series . insert ( s . T , 0 , nil , s . FH , a . head . chunkDiskMapper , oooCapMax , a . head . logger )
if chunkCreated {
r , ok := oooMmapMarkers [ series . ref ]
if ! ok || r != nil {
// !ok means there are no markers collected for these samples yet. So we first flush the samples
// before setting this m-map marker.
// r != 0 means we have already m-mapped a chunk for this series in the same Commit().
// Hence, before we m-map again, we should add the samples and m-map markers
// seen till now to the WBL records.
collectOOORecords ( )
}
if oooMmapMarkers == nil {
oooMmapMarkers = make ( map [ chunks . HeadSeriesRef ] [ ] chunks . ChunkDiskMapperRef )
}
if len ( mmapRefs ) > 0 {
oooMmapMarkers [ series . ref ] = mmapRefs
oooMmapMarkersCount += len ( mmapRefs )
} else {
// No chunk was written to disk, so we need to set an initial marker for this series.
oooMmapMarkers [ series . ref ] = [ ] chunks . ChunkDiskMapperRef { 0 }
oooMmapMarkersCount ++
}
}
if ok {
wblFloatHistograms = append ( wblFloatHistograms , s )
if s . T < oooMinT {
oooMinT = s . T
}
if s . T > oooMaxT {
oooMaxT = s . T
}
oooHistogramAccepted ++
} else {
// Sample is an exact duplicate of the last sample.
// NOTE: We can only detect updates if they clash with a sample in the OOOHeadChunk,
// not with samples in already flushed OOO chunks.
// TODO(codesome): Add error reporting? It depends on addressing https://github.com/prometheus/prometheus/discussions/10305.
histogramsAppended --
}
default :
ok , chunkCreated = series . appendFloatHistogram ( s . T , s . FH , a . appendID , appendChunkOpts )
if ok {
if s . T < inOrderMint {
inOrderMint = s . T
}
if s . T > inOrderMaxt {
inOrderMaxt = s . T
}
} else {
histogramsAppended --
histoOOORejected ++
}
}
if chunkCreated {
a . head . metrics . chunks . Inc ( )
a . head . metrics . chunksCreated . Inc ( )
}
series . cleanupAppendIDsBelow ( a . cleanupAppendIDsBelow )
series . pendingCommit = false
series . Unlock ( )
}
for i , m := range a . metadata {
@ -1067,6 +1299,7 @@ func (a *headAppender) Commit() (err error) {
a . head . metrics . samplesAppended . WithLabelValues ( sampleMetricTypeFloat ) . Add ( float64 ( floatsAppended ) )
a . head . metrics . samplesAppended . WithLabelValues ( sampleMetricTypeHistogram ) . Add ( float64 ( histogramsAppended ) )
a . head . metrics . outOfOrderSamplesAppended . WithLabelValues ( sampleMetricTypeFloat ) . Add ( float64 ( oooFloatsAccepted ) )
a . head . metrics . outOfOrderSamplesAppended . WithLabelValues ( sampleMetricTypeHistogram ) . Add ( float64 ( oooHistogramAccepted ) )
a . head . updateMinMaxTime ( inOrderMint , inOrderMaxt )
a . head . updateMinOOOMaxOOOTime ( oooMinT , oooMaxT )
Carrie Edwards
2 months ago
committed by