2022-12-20 10:03:32 +00:00
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chunkenc
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/prometheus/prometheus/model/histogram"
2023-11-29 10:39:12 +00:00
"github.com/prometheus/prometheus/tsdb/tsdbutil"
2022-12-20 10:03:32 +00:00
)
type floatResult struct {
t int64
h * histogram . FloatHistogram
}
2023-09-01 21:39:15 +00:00
func TestFirstFloatHistogramExplicitCounterReset ( t * testing . T ) {
tests := map [ string ] struct {
hint histogram . CounterResetHint
expHeader CounterResetHeader
} {
"CounterReset" : {
hint : histogram . CounterReset ,
expHeader : CounterReset ,
} ,
"NotCounterReset" : {
hint : histogram . NotCounterReset ,
expHeader : UnknownCounterReset ,
} ,
"UnknownCounterReset" : {
hint : histogram . UnknownCounterReset ,
expHeader : UnknownCounterReset ,
} ,
"Gauge" : {
hint : histogram . GaugeType ,
expHeader : GaugeType ,
} ,
}
for name , test := range tests {
t . Run ( name , func ( t * testing . T ) {
h := & histogram . FloatHistogram {
CounterResetHint : test . hint ,
}
chk := NewFloatHistogramChunk ( )
app , err := chk . Appender ( )
require . NoError ( t , err )
newChk , recoded , newApp , err := app . AppendFloatHistogram ( nil , 0 , h , false )
require . NoError ( t , err )
require . Nil ( t , newChk )
require . False ( t , recoded )
require . Equal ( t , app , newApp )
require . Equal ( t , test . expHeader , chk . GetCounterResetHeader ( ) )
} )
}
}
2022-12-20 10:03:32 +00:00
func TestFloatHistogramChunkSameBuckets ( t * testing . T ) {
c := NewFloatHistogramChunk ( )
var exp [ ] floatResult
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
require . Equal ( t , 0 , c . NumSamples ( ) )
ts := int64 ( 1234567890 )
h := & histogram . Histogram {
Count : 15 ,
ZeroCount : 2 ,
Sum : 18.4 ,
ZeroThreshold : 1e-100 ,
Schema : 1 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 1 , Length : 2 } ,
} ,
PositiveBuckets : [ ] int64 { 1 , 1 , - 1 , 0 } , // counts: 1, 2, 1, 1 (total 5)
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 1 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] int64 { 2 , 1 , - 1 , - 1 } , // counts: 2, 3, 2, 1 (total 8)
}
2023-11-29 14:15:57 +00:00
chk , _ , app , err := app . AppendFloatHistogram ( nil , ts , h . ToFloat ( nil ) , false )
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , chk )
2023-11-29 14:15:57 +00:00
exp = append ( exp , floatResult { t : ts , h : h . ToFloat ( nil ) } )
2022-12-20 10:03:32 +00:00
require . Equal ( t , 1 , c . NumSamples ( ) )
// Add an updated histogram.
ts += 16
h = h . Copy ( )
h . Count = 32
h . ZeroCount ++
h . Sum = 24.4
h . PositiveBuckets = [ ] int64 { 5 , - 2 , 1 , - 2 } // counts: 5, 3, 4, 2 (total 14)
h . NegativeBuckets = [ ] int64 { 4 , - 1 , 1 , - 1 } // counts: 4, 3, 4, 4 (total 15)
2023-11-29 14:15:57 +00:00
chk , _ , _ , err = app . AppendFloatHistogram ( nil , ts , h . ToFloat ( nil ) , false )
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , chk )
2023-11-29 14:15:57 +00:00
expH := h . ToFloat ( nil )
2023-01-18 16:59:29 +00:00
expH . CounterResetHint = histogram . NotCounterReset
exp = append ( exp , floatResult { t : ts , h : expH } )
2022-12-20 10:03:32 +00:00
require . Equal ( t , 2 , c . NumSamples ( ) )
// Add update with new appender.
app , err = c . Appender ( )
require . NoError ( t , err )
ts += 14
h = h . Copy ( )
h . Count = 54
h . ZeroCount += 2
h . Sum = 24.4
h . PositiveBuckets = [ ] int64 { 6 , 1 , - 3 , 6 } // counts: 6, 7, 4, 10 (total 27)
h . NegativeBuckets = [ ] int64 { 5 , 1 , - 2 , 3 } // counts: 5, 6, 4, 7 (total 22)
2023-11-29 14:15:57 +00:00
chk , _ , _ , err = app . AppendFloatHistogram ( nil , ts , h . ToFloat ( nil ) , false )
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , chk )
2023-11-29 14:15:57 +00:00
expH = h . ToFloat ( nil )
2023-01-18 16:59:29 +00:00
expH . CounterResetHint = histogram . NotCounterReset
exp = append ( exp , floatResult { t : ts , h : expH } )
2022-12-20 10:03:32 +00:00
require . Equal ( t , 3 , c . NumSamples ( ) )
// 1. Expand iterator in simple case.
it := c . Iterator ( nil )
require . NoError ( t , it . Err ( ) )
var act [ ] floatResult
for it . Next ( ) == ValFloatHistogram {
2024-01-23 16:02:14 +00:00
fts , fh := it . AtFloatHistogram ( nil )
2022-12-20 10:03:32 +00:00
act = append ( act , floatResult { t : fts , h : fh } )
}
require . NoError ( t , it . Err ( ) )
require . Equal ( t , exp , act )
// 2. Expand second iterator while reusing first one.
it2 := c . Iterator ( it )
var act2 [ ] floatResult
for it2 . Next ( ) == ValFloatHistogram {
2024-01-23 16:02:14 +00:00
fts , fh := it2 . AtFloatHistogram ( nil )
2022-12-20 10:03:32 +00:00
act2 = append ( act2 , floatResult { t : fts , h : fh } )
}
require . NoError ( t , it2 . Err ( ) )
require . Equal ( t , exp , act2 )
// 3. Now recycle an iterator that was never used to access anything.
itX := c . Iterator ( nil )
2023-10-31 11:35:13 +00:00
for itX . Next ( ) == ValFloatHistogram {
2022-12-20 10:03:32 +00:00
// Just iterate through without accessing anything.
}
it3 := c . iterator ( itX )
var act3 [ ] floatResult
for it3 . Next ( ) == ValFloatHistogram {
2024-01-23 16:02:14 +00:00
fts , fh := it3 . AtFloatHistogram ( nil )
2022-12-20 10:03:32 +00:00
act3 = append ( act3 , floatResult { t : fts , h : fh } )
}
require . NoError ( t , it3 . Err ( ) )
require . Equal ( t , exp , act3 )
// 4. Test iterator Seek.
mid := len ( exp ) / 2
it4 := c . Iterator ( nil )
var act4 [ ] floatResult
require . Equal ( t , ValFloatHistogram , it4 . Seek ( exp [ mid ] . t ) )
// Below ones should not matter.
require . Equal ( t , ValFloatHistogram , it4 . Seek ( exp [ mid ] . t ) )
require . Equal ( t , ValFloatHistogram , it4 . Seek ( exp [ mid ] . t ) )
2024-01-23 16:02:14 +00:00
fts , fh := it4 . AtFloatHistogram ( nil )
2022-12-20 10:03:32 +00:00
act4 = append ( act4 , floatResult { t : fts , h : fh } )
for it4 . Next ( ) == ValFloatHistogram {
2024-01-23 16:02:14 +00:00
fts , fh := it4 . AtFloatHistogram ( nil )
2022-12-20 10:03:32 +00:00
act4 = append ( act4 , floatResult { t : fts , h : fh } )
}
require . NoError ( t , it4 . Err ( ) )
require . Equal ( t , exp [ mid : ] , act4 )
require . Equal ( t , ValNone , it4 . Seek ( exp [ len ( exp ) - 1 ] . t + 1 ) )
}
2023-01-18 12:47:22 +00:00
// Mimics the scenario described for expandSpansForward.
2022-12-20 10:03:32 +00:00
func TestFloatHistogramChunkBucketChanges ( t * testing . T ) {
c := Chunk ( NewFloatHistogramChunk ( ) )
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
require . Equal ( t , 0 , c . NumSamples ( ) )
ts1 := int64 ( 1234567890 )
h1 := & histogram . Histogram {
Count : 27 ,
ZeroCount : 2 ,
Sum : 18.4 ,
ZeroThreshold : 1e-125 ,
Schema : 1 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
} ,
PositiveBuckets : [ ] int64 { 6 , - 3 , 0 , - 1 , 2 , 1 , - 4 } , // counts: 6, 3, 3, 2, 4, 5, 1 (total 24)
NegativeSpans : [ ] histogram . Span { { Offset : 1 , Length : 1 } } ,
NegativeBuckets : [ ] int64 { 1 } ,
}
2023-11-29 14:15:57 +00:00
chk , _ , app , err := app . AppendFloatHistogram ( nil , ts1 , h1 . ToFloat ( nil ) , false )
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , chk )
2022-12-20 10:03:32 +00:00
require . Equal ( t , 1 , c . NumSamples ( ) )
// Add a new histogram that has expanded buckets.
ts2 := ts1 + 16
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . NegativeSpans = [ ] histogram . Span { { Offset : 0 , Length : 2 } }
h2 . Count = 35
h2 . ZeroCount ++
h2 . Sum = 30
// Existing histogram should get values converted from the above to:
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2 . PositiveBuckets = [ ] int64 { 7 , - 2 , - 4 , 2 , - 2 , - 1 , 2 , 3 , 0 , - 5 , 1 } // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
// Existing histogram should get values converted from the above to:
// 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2 . NegativeBuckets = [ ] int64 { 2 , - 1 } // 2 1 (total 3)
// This is how span changes will be handled.
hApp , _ := app . ( * FloatHistogramAppender )
2024-08-01 07:22:32 +00:00
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 . ToFloat ( nil ) )
2023-12-07 11:35:01 +00:00
require . NotEmpty ( t , posInterjections )
require . NotEmpty ( t , negInterjections )
2024-08-01 07:22:32 +00:00
require . Empty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
2022-12-20 10:03:32 +00:00
require . True ( t , ok ) // Only new buckets came in.
require . False ( t , cr )
2023-07-26 13:08:16 +00:00
c , app = hApp . recode ( posInterjections , negInterjections , h2 . PositiveSpans , h2 . NegativeSpans )
2023-11-29 14:15:57 +00:00
chk , _ , _ , err = app . AppendFloatHistogram ( nil , ts2 , h2 . ToFloat ( nil ) , false )
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , chk )
2022-12-20 10:03:32 +00:00
require . Equal ( t , 2 , c . NumSamples ( ) )
// Because the 2nd histogram has expanded buckets, we should expect all
// histograms (in particular the first) to come back using the new spans
// metadata as well as the expanded buckets.
h1 . PositiveSpans = h2 . PositiveSpans
h1 . PositiveBuckets = [ ] int64 { 6 , - 3 , - 3 , 3 , - 3 , 0 , 2 , 2 , 1 , - 5 , 1 }
h1 . NegativeSpans = h2 . NegativeSpans
h1 . NegativeBuckets = [ ] int64 { 0 , 1 }
2023-11-29 14:15:57 +00:00
expH2 := h2 . ToFloat ( nil )
2023-01-18 16:59:29 +00:00
expH2 . CounterResetHint = histogram . NotCounterReset
2022-12-20 10:03:32 +00:00
exp := [ ] floatResult {
2023-11-29 14:15:57 +00:00
{ t : ts1 , h : h1 . ToFloat ( nil ) } ,
2023-01-18 16:59:29 +00:00
{ t : ts2 , h : expH2 } ,
2022-12-20 10:03:32 +00:00
}
it := c . Iterator ( nil )
var act [ ] floatResult
for it . Next ( ) == ValFloatHistogram {
2024-01-23 16:02:14 +00:00
fts , fh := it . AtFloatHistogram ( nil )
2022-12-20 10:03:32 +00:00
act = append ( act , floatResult { t : fts , h : fh } )
}
require . NoError ( t , it . Err ( ) )
require . Equal ( t , exp , act )
}
func TestFloatHistogramChunkAppendable ( t * testing . T ) {
2024-03-22 13:36:39 +00:00
eh := & histogram . FloatHistogram {
Count : 5 ,
ZeroCount : 2 ,
Sum : 18.4 ,
ZeroThreshold : 1e-125 ,
Schema : 1 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
} ,
PositiveBuckets : [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 1 } ,
}
cbh := & histogram . FloatHistogram {
Count : 24 ,
Sum : 18.4 ,
Schema : histogram . CustomBucketsSchema ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
} ,
PositiveBuckets : [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 1 } ,
CustomValues : [ ] float64 { 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 } ,
}
setup := func ( h * histogram . FloatHistogram ) ( Chunk , * FloatHistogramAppender , int64 , * histogram . FloatHistogram ) {
2023-07-26 13:08:16 +00:00
c := Chunk ( NewFloatHistogramChunk ( ) )
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
require . Equal ( t , 0 , c . NumSamples ( ) )
ts := int64 ( 1234567890 )
2022-12-20 10:03:32 +00:00
2024-03-22 13:36:39 +00:00
chk , _ , app , err := app . AppendFloatHistogram ( nil , ts , h . Copy ( ) , false )
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , chk )
require . Equal ( t , 1 , c . NumSamples ( ) )
require . Equal ( t , UnknownCounterReset , c . ( * FloatHistogramChunk ) . GetCounterResetHeader ( ) )
2024-03-22 13:36:39 +00:00
return c , app . ( * FloatHistogramAppender ) , ts , h
2022-12-20 10:03:32 +00:00
}
2022-12-28 15:37:41 +00:00
{ // Schema change.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2022-12-28 15:37:41 +00:00
h2 := h1 . Copy ( )
h2 . Schema ++
2024-08-01 07:22:32 +00:00
_ , _ , _ , _ , ok , _ := hApp . appendable ( h2 )
2022-12-28 15:37:41 +00:00
require . False ( t , ok )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
2022-12-28 15:37:41 +00:00
}
{ // Zero threshold change.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2022-12-28 15:37:41 +00:00
h2 := h1 . Copy ( )
h2 . ZeroThreshold += 0.1
2024-08-01 07:22:32 +00:00
_ , _ , _ , _ , ok , _ := hApp . appendable ( h2 )
2022-12-28 15:37:41 +00:00
require . False ( t , ok )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
2022-12-28 15:37:41 +00:00
}
2022-12-20 10:03:32 +00:00
{ // New histogram that has more buckets.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2022-12-28 15:37:41 +00:00
h2 := h1 . Copy ( )
2022-12-20 10:03:32 +00:00
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . Count += 9
h2 . ZeroCount ++
h2 . Sum = 30
2022-12-28 15:37:41 +00:00
h2 . PositiveBuckets = [ ] float64 { 7 , 5 , 1 , 3 , 1 , 0 , 2 , 5 , 5 , 0 , 1 }
2022-12-20 10:03:32 +00:00
2024-08-01 07:22:32 +00:00
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
2023-12-07 11:35:01 +00:00
require . NotEmpty ( t , posInterjections )
require . Empty ( t , negInterjections )
2024-08-01 07:22:32 +00:00
require . Empty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
2022-12-20 10:03:32 +00:00
require . True ( t , ok ) // Only new buckets came in.
require . False ( t , cr )
2023-07-26 13:08:16 +00:00
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
2022-12-20 10:03:32 +00:00
}
{ // New histogram that has a bucket missing.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-07-26 13:08:16 +00:00
2022-12-28 15:37:41 +00:00
h2 := h1 . Copy ( )
2022-12-20 10:03:32 +00:00
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 5 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
}
h2 . Sum = 21
2022-12-28 15:37:41 +00:00
h2 . PositiveBuckets = [ ] float64 { 6 , 3 , 2 , 4 , 5 , 1 }
2022-12-20 10:03:32 +00:00
2024-08-01 07:22:32 +00:00
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
2023-12-07 11:35:01 +00:00
require . Empty ( t , posInterjections )
require . Empty ( t , negInterjections )
2024-08-01 07:22:32 +00:00
require . Empty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
2022-12-20 10:03:32 +00:00
require . False ( t , ok ) // Need to cut a new chunk.
require . True ( t , cr )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , CounterReset )
2022-12-20 10:03:32 +00:00
}
2024-08-01 07:22:32 +00:00
{ // New histogram that has buckets missing but the buckets missing were empty.
emptyBucketH := eh . Copy ( )
emptyBucketH . PositiveBuckets = [ ] float64 { 6 , 0 , 3 , 2 , 4 , 0 , 1 }
c , hApp , ts , h1 := setup ( emptyBucketH )
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 1 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 5 , Length : 1 } ,
}
2024-08-06 14:51:20 +00:00
savedH2Spans := h2 . PositiveSpans
2024-08-01 07:22:32 +00:00
h2 . PositiveBuckets = [ ] float64 { 7 , 4 , 3 , 5 , 2 }
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
require . Empty ( t , posInterjections )
require . Empty ( t , negInterjections )
require . NotEmpty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
require . True ( t , ok )
require . False ( t , cr )
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
// Check that h2 was recoded.
require . Equal ( t , [ ] float64 { 7 , 0 , 4 , 3 , 5 , 0 , 2 } , h2 . PositiveBuckets )
require . Equal ( t , emptyBucketH . PositiveSpans , h2 . PositiveSpans )
2024-08-06 14:51:20 +00:00
require . NotEqual ( t , savedH2Spans , h2 . PositiveSpans , "recoding must make a copy" )
2024-08-01 07:22:32 +00:00
}
2024-08-06 08:51:44 +00:00
{ // New histogram that has new buckets AND buckets missing but the buckets missing were empty.
emptyBucketH := eh . Copy ( )
emptyBucketH . PositiveBuckets = [ ] float64 { 6 , 0 , 3 , 2 , 4 , 0 , 1 }
c , hApp , ts , h1 := setup ( emptyBucketH )
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 1 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 5 , Length : 2 } ,
}
2024-08-06 14:51:20 +00:00
savedH2Spans := h2 . PositiveSpans
2024-08-06 08:51:44 +00:00
h2 . PositiveBuckets = [ ] float64 { 7 , 4 , 3 , 5 , 2 , 3 }
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
require . NotEmpty ( t , posInterjections )
require . Empty ( t , negInterjections )
require . NotEmpty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
require . True ( t , ok )
require . False ( t , cr )
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
// Check that h2 was recoded.
require . Equal ( t , [ ] float64 { 7 , 0 , 4 , 3 , 5 , 0 , 2 , 3 } , h2 . PositiveBuckets )
2024-08-06 11:08:10 +00:00
require . Equal ( t , [ ] histogram . Span {
{ Offset : 0 , Length : 2 } , // Added empty bucket.
{ Offset : 2 , Length : 1 } , // Existing - offset adjusted.
{ Offset : 3 , Length : 2 } , // Existing.
{ Offset : 3 , Length : 1 } , // Added empty bucket.
{ Offset : 1 , Length : 2 } , // Existing + the extra bucket.
} , h2 . PositiveSpans )
2024-08-06 14:51:20 +00:00
require . NotEqual ( t , savedH2Spans , h2 . PositiveSpans , "recoding must make a copy" )
2024-08-06 08:51:44 +00:00
}
2022-12-20 10:03:32 +00:00
{ // New histogram that has a counter reset while buckets are same.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2022-12-28 15:37:41 +00:00
h2 := h1 . Copy ( )
2022-12-20 10:03:32 +00:00
h2 . Sum = 23
2022-12-28 15:37:41 +00:00
h2 . PositiveBuckets = [ ] float64 { 6 , 2 , 3 , 2 , 4 , 5 , 1 }
2022-12-20 10:03:32 +00:00
2024-08-01 07:22:32 +00:00
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
2023-12-07 11:35:01 +00:00
require . Empty ( t , posInterjections )
require . Empty ( t , negInterjections )
2024-08-01 07:22:32 +00:00
require . Empty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
2022-12-20 10:03:32 +00:00
require . False ( t , ok ) // Need to cut a new chunk.
require . True ( t , cr )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , CounterReset )
2022-12-20 10:03:32 +00:00
}
{ // New histogram that has a counter reset while new buckets were added.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2022-12-28 15:37:41 +00:00
h2 := h1 . Copy ( )
2022-12-20 10:03:32 +00:00
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . Sum = 29
2022-12-28 15:37:41 +00:00
h2 . PositiveBuckets = [ ] float64 { 7 , 5 , 1 , 3 , 1 , 0 , 2 , 5 , 5 , 0 , 0 }
2022-12-20 10:03:32 +00:00
2024-08-01 07:22:32 +00:00
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
2023-12-07 11:35:01 +00:00
require . Empty ( t , posInterjections )
require . Empty ( t , negInterjections )
2024-08-01 07:22:32 +00:00
require . Empty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
2022-12-20 10:03:32 +00:00
require . False ( t , ok ) // Need to cut a new chunk.
require . True ( t , cr )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , CounterReset )
2022-12-20 10:03:32 +00:00
}
{
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2022-12-20 10:03:32 +00:00
// New histogram that has a counter reset while new buckets were
// added before the first bucket and reset on first bucket. (to
// catch the edge case where the new bucket should be forwarded
// ahead until first old bucket at start)
2022-12-28 15:37:41 +00:00
h2 := h1 . Copy ( )
2022-12-20 10:03:32 +00:00
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : - 3 , Length : 2 } ,
{ Offset : 1 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
}
h2 . Sum = 26
2022-12-28 15:37:41 +00:00
h2 . PositiveBuckets = [ ] float64 { 1 , 2 , 5 , 3 , 3 , 2 , 4 , 5 , 1 }
2022-12-20 10:03:32 +00:00
2024-08-01 07:22:32 +00:00
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
2023-12-07 11:35:01 +00:00
require . Empty ( t , posInterjections )
require . Empty ( t , negInterjections )
2024-08-01 07:22:32 +00:00
require . Empty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
2022-12-20 10:03:32 +00:00
require . False ( t , ok ) // Need to cut a new chunk.
require . True ( t , cr )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , CounterReset )
2022-12-20 10:03:32 +00:00
}
2023-09-01 21:39:15 +00:00
{ // New histogram that has an explicit counter reset.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-09-01 21:39:15 +00:00
h2 := h1 . Copy ( )
h2 . CounterResetHint = histogram . CounterReset
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , CounterReset )
}
{ // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk.
2024-03-22 13:36:39 +00:00
_ , hApp , ts , h1 := setup ( eh )
2023-09-01 21:39:15 +00:00
h2 := h1 . Copy ( ) // Identity is appendable.
nextChunk := NewFloatHistogramChunk ( )
app , err := nextChunk . Appender ( )
require . NoError ( t , err )
newChunk , recoded , newApp , err := app . AppendFloatHistogram ( hApp , ts + 1 , h2 , false )
require . NoError ( t , err )
require . Nil ( t , newChunk )
require . False ( t , recoded )
require . Equal ( t , app , newApp )
assertSampleCount ( t , nextChunk , 1 , ValFloatHistogram )
require . Equal ( t , NotCounterReset , nextChunk . GetCounterResetHeader ( ) )
}
{ // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk.
2024-03-22 13:36:39 +00:00
_ , hApp , ts , h1 := setup ( eh )
2023-09-01 21:39:15 +00:00
h2 := h1 . Copy ( )
h2 . Count -- // Make this not appendable due to counter reset.
nextChunk := NewFloatHistogramChunk ( )
app , err := nextChunk . Appender ( )
require . NoError ( t , err )
newChunk , recoded , newApp , err := app . AppendFloatHistogram ( hApp , ts + 1 , h2 , false )
require . NoError ( t , err )
require . Nil ( t , newChunk )
require . False ( t , recoded )
require . Equal ( t , app , newApp )
assertSampleCount ( t , nextChunk , 1 , ValFloatHistogram )
require . Equal ( t , CounterReset , nextChunk . GetCounterResetHeader ( ) )
}
{ // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk.
2024-03-22 13:36:39 +00:00
_ , hApp , ts , h1 := setup ( eh )
2023-09-01 21:39:15 +00:00
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . Count += 9
h2 . ZeroCount ++
h2 . Sum = 30
h2 . PositiveBuckets = [ ] float64 { 7 , 5 , 1 , 3 , 1 , 0 , 2 , 5 , 5 , 0 , 1 }
nextChunk := NewFloatHistogramChunk ( )
app , err := nextChunk . Appender ( )
require . NoError ( t , err )
newChunk , recoded , newApp , err := app . AppendFloatHistogram ( hApp , ts + 1 , h2 , false )
require . NoError ( t , err )
require . Nil ( t , newChunk )
require . False ( t , recoded )
require . Equal ( t , app , newApp )
assertSampleCount ( t , nextChunk , 1 , ValFloatHistogram )
require . Equal ( t , NotCounterReset , nextChunk . GetCounterResetHeader ( ) )
}
2024-03-22 13:36:39 +00:00
2024-08-01 07:22:32 +00:00
{
// Start a new chunk with a histogram that has an empty bucket.
// Add a histogram that has the same bucket missing.
// This should be appendable and can happen if we are merging from chunks
// where the first sample came from a recoded chunk that added the
// empty bucket.
h1 := eh . Copy ( )
// Add a bucket that is empty -10 offsets from the first bucket.
h1 . PositiveSpans = make ( [ ] histogram . Span , len ( eh . PositiveSpans ) + 1 )
h1 . PositiveSpans [ 0 ] = histogram . Span { Offset : eh . PositiveSpans [ 0 ] . Offset - 10 , Length : 1 }
h1 . PositiveSpans [ 1 ] = histogram . Span { Offset : eh . PositiveSpans [ 0 ] . Offset + 9 , Length : eh . PositiveSpans [ 0 ] . Length }
for i , v := range eh . PositiveSpans [ 1 : ] {
h1 . PositiveSpans [ i + 2 ] = v
}
h1 . PositiveBuckets = make ( [ ] float64 , len ( eh . PositiveBuckets ) + 1 )
h1 . PositiveBuckets [ 0 ] = 0
for i , v := range eh . PositiveBuckets {
h1 . PositiveBuckets [ i + 1 ] = v
}
c , hApp , ts , _ := setup ( h1 )
h2 := eh . Copy ( )
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
require . Empty ( t , posInterjections )
require . Empty ( t , negInterjections )
require . NotEmpty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
require . True ( t , ok )
require . False ( t , cr )
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
}
2024-03-22 13:36:39 +00:00
{ // Custom buckets, no change.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
2024-08-01 07:22:32 +00:00
_ , _ , _ , _ , ok , _ := hApp . appendable ( h2 )
2024-03-22 13:36:39 +00:00
require . True ( t , ok )
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
}
{ // Custom buckets, increase in bucket counts but no change in layout.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . Count ++
h2 . PositiveBuckets = [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 2 }
2024-08-01 07:22:32 +00:00
_ , _ , _ , _ , ok , _ := hApp . appendable ( h2 )
2024-03-22 13:36:39 +00:00
require . True ( t , ok )
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
}
{ // Custom buckets, decrease in bucket counts but no change in layout.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . Count --
h2 . PositiveBuckets = [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 0 }
2024-08-01 07:22:32 +00:00
_ , _ , _ , _ , ok , _ := hApp . appendable ( h2 )
2024-03-22 13:36:39 +00:00
require . False ( t , ok )
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , CounterReset )
}
{ // Custom buckets, change only in custom bounds.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . CustomValues = [ ] float64 { 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 }
2024-08-01 07:22:32 +00:00
_ , _ , _ , _ , ok , _ := hApp . appendable ( h2 )
2024-03-22 13:36:39 +00:00
require . False ( t , ok )
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , CounterReset )
}
{ // Custom buckets, with more buckets.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . Count += 6
h2 . Sum = 30
// Existing histogram should get values converted from the above to:
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2 . PositiveBuckets = [ ] float64 { 7 , 5 , 1 , 3 , 1 , 0 , 2 , 5 , 5 , 0 , 1 } // (total 30)
2024-08-01 07:22:32 +00:00
posInterjections , negInterjections , backwardPositiveInserts , backwardNegativeInserts , ok , cr := hApp . appendable ( h2 )
2024-03-22 13:36:39 +00:00
require . NotEmpty ( t , posInterjections )
require . Empty ( t , negInterjections )
2024-08-01 07:22:32 +00:00
require . Empty ( t , backwardPositiveInserts )
require . Empty ( t , backwardNegativeInserts )
2024-03-22 13:36:39 +00:00
require . True ( t , ok ) // Only new buckets came in.
require . False ( t , cr )
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , UnknownCounterReset )
}
2022-12-20 10:03:32 +00:00
}
2023-01-04 10:00:06 +00:00
2023-07-26 13:08:16 +00:00
func assertNewFloatHistogramChunkOnAppend ( t * testing . T , oldChunk Chunk , hApp * FloatHistogramAppender , ts int64 , h * histogram . FloatHistogram , expectHeader CounterResetHeader ) {
oldChunkBytes := oldChunk . Bytes ( )
newChunk , recoded , newAppender , err := hApp . AppendFloatHistogram ( nil , ts , h , false )
require . Equal ( t , oldChunkBytes , oldChunk . Bytes ( ) ) // Sanity check that previous chunk is untouched.
require . NoError ( t , err )
require . NotNil ( t , newChunk )
require . False ( t , recoded )
require . NotEqual ( t , oldChunk , newChunk )
require . Equal ( t , expectHeader , newChunk . ( * FloatHistogramChunk ) . GetCounterResetHeader ( ) )
require . NotNil ( t , newAppender )
require . NotEqual ( t , hApp , newAppender )
assertSampleCount ( t , newChunk , 1 , ValFloatHistogram )
}
func assertNoNewFloatHistogramChunkOnAppend ( t * testing . T , oldChunk Chunk , hApp * FloatHistogramAppender , ts int64 , h * histogram . FloatHistogram , expectHeader CounterResetHeader ) {
oldChunkBytes := oldChunk . Bytes ( )
newChunk , recoded , newAppender , err := hApp . AppendFloatHistogram ( nil , ts , h , false )
2024-03-22 13:36:39 +00:00
require . Greater ( t , len ( oldChunk . Bytes ( ) ) , len ( oldChunkBytes ) ) // Check that current chunk is bigger than previously.
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , newChunk )
require . False ( t , recoded )
require . Equal ( t , expectHeader , oldChunk . ( * FloatHistogramChunk ) . GetCounterResetHeader ( ) )
require . NotNil ( t , newAppender )
require . Equal ( t , hApp , newAppender )
assertSampleCount ( t , oldChunk , 2 , ValFloatHistogram )
}
func assertRecodedFloatHistogramChunkOnAppend ( t * testing . T , prevChunk Chunk , hApp * FloatHistogramAppender , ts int64 , h * histogram . FloatHistogram , expectHeader CounterResetHeader ) {
prevChunkBytes := prevChunk . Bytes ( )
newChunk , recoded , newAppender , err := hApp . AppendFloatHistogram ( nil , ts , h , false )
require . Equal ( t , prevChunkBytes , prevChunk . Bytes ( ) ) // Sanity check that previous chunk is untouched. This may change in the future if we implement in-place recoding.
require . NoError ( t , err )
require . NotNil ( t , newChunk )
require . True ( t , recoded )
require . NotEqual ( t , prevChunk , newChunk )
require . Equal ( t , expectHeader , newChunk . ( * FloatHistogramChunk ) . GetCounterResetHeader ( ) )
require . NotNil ( t , newAppender )
require . NotEqual ( t , hApp , newAppender )
assertSampleCount ( t , newChunk , 2 , ValFloatHistogram )
}
2023-05-11 19:02:02 +00:00
func TestFloatHistogramChunkAppendableWithEmptySpan ( t * testing . T ) {
2023-09-13 10:59:42 +00:00
tests := map [ string ] struct {
h1 * histogram . FloatHistogram
h2 * histogram . FloatHistogram
} {
"empty span in old and new histogram" : {
h1 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 21 ,
Sum : 1234.5 ,
ZeroThreshold : 0.001 ,
ZeroCount : 4 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 0 , Length : 0 } ,
{ Offset : 0 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 2 , 1 , 1 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 2 , 1 , 2 , 2 , 2 , 2 } ,
} ,
h2 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 37 ,
Sum : 2345.6 ,
ZeroThreshold : 0.001 ,
ZeroCount : 5 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 0 , Length : 0 } ,
{ Offset : 0 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 3 , 1 , 2 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 4 , 2 , 7 , 5 , 5 , 2 } ,
} ,
2023-05-11 19:02:02 +00:00
} ,
2023-09-13 10:59:42 +00:00
"empty span in old histogram" : {
h1 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 21 ,
Sum : 1234.5 ,
ZeroThreshold : 0.001 ,
ZeroCount : 4 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 0 } , // This span will disappear.
{ Offset : 2 , Length : 4 } ,
{ Offset : 0 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 2 , 1 , 1 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 2 , 1 , 2 , 2 , 2 , 2 } ,
} ,
h2 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 37 ,
Sum : 2345.6 ,
ZeroThreshold : 0.001 ,
ZeroCount : 5 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 3 , Length : 4 } ,
{ Offset : 0 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 3 , 1 , 2 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 4 , 2 , 7 , 5 , 5 , 2 } ,
} ,
2023-05-11 19:02:02 +00:00
} ,
2023-09-13 10:59:42 +00:00
"empty span in new histogram" : {
h1 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 21 ,
Sum : 1234.5 ,
ZeroThreshold : 0.001 ,
ZeroCount : 4 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 2 , 1 , 1 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 2 , 1 , 2 , 2 , 2 , 2 } ,
} ,
h2 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 37 ,
Sum : 2345.6 ,
ZeroThreshold : 0.001 ,
ZeroCount : 5 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 1 , Length : 0 } , // This span is new.
{ Offset : 2 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 3 , 1 , 2 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 4 , 2 , 7 , 5 , 5 , 2 } ,
} ,
2023-05-11 19:02:02 +00:00
} ,
2023-09-13 10:59:42 +00:00
"two empty spans mixing offsets" : {
h1 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 21 ,
Sum : 1234.5 ,
ZeroThreshold : 0.001 ,
ZeroCount : 4 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 1 , Length : 0 } ,
{ Offset : 3 , Length : 0 } ,
{ Offset : 4 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 2 , 1 , 1 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 2 , 1 , 2 , 2 , 2 , 2 } ,
} ,
h2 : & histogram . FloatHistogram {
Schema : 0 ,
Count : 37 ,
Sum : 2345.6 ,
ZeroThreshold : 0.001 ,
ZeroCount : 5 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 3 , Length : 0 } ,
{ Offset : 1 , Length : 0 } ,
{ Offset : 4 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 3 , 1 , 2 , 1 , 1 , 1 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 4 } ,
{ Offset : 2 , Length : 0 } ,
{ Offset : 2 , Length : 3 } ,
} ,
NegativeBuckets : [ ] float64 { 1 , 4 , 2 , 7 , 5 , 5 , 2 } ,
} ,
2023-05-11 19:02:02 +00:00
} ,
2024-03-22 13:36:39 +00:00
"empty span in old and new custom buckets histogram" : {
h1 : & histogram . FloatHistogram {
Schema : histogram . CustomBucketsSchema ,
Count : 7 ,
Sum : 1234.5 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 0 , Length : 0 } ,
{ Offset : 0 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 2 , 1 , 1 , 1 , 1 , 1 } ,
CustomValues : [ ] float64 { 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 } ,
} ,
h2 : & histogram . FloatHistogram {
Schema : histogram . CustomBucketsSchema ,
Count : 10 ,
Sum : 2345.6 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 4 } ,
{ Offset : 0 , Length : 0 } ,
{ Offset : 0 , Length : 3 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 3 , 1 , 2 , 1 , 1 , 1 } ,
CustomValues : [ ] float64 { 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 } ,
} ,
} ,
2023-05-11 19:02:02 +00:00
}
2023-09-13 10:59:42 +00:00
for name , tc := range tests {
t . Run ( name , func ( t * testing . T ) {
c := Chunk ( NewFloatHistogramChunk ( ) )
2023-05-11 19:02:02 +00:00
2023-09-13 10:59:42 +00:00
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
require . Equal ( t , 0 , c . NumSamples ( ) )
2023-05-11 19:02:02 +00:00
2023-09-13 10:59:42 +00:00
_ , _ , _ , err = app . AppendFloatHistogram ( nil , 1 , tc . h1 , true )
require . NoError ( t , err )
require . Equal ( t , 1 , c . NumSamples ( ) )
hApp , _ := app . ( * FloatHistogramAppender )
2024-08-01 07:22:32 +00:00
pI , nI , bpI , bnI , okToAppend , counterReset := hApp . appendable ( tc . h2 )
2023-09-13 10:59:42 +00:00
require . Empty ( t , pI )
require . Empty ( t , nI )
2024-08-01 07:22:32 +00:00
require . Empty ( t , bpI )
require . Empty ( t , bnI )
2023-09-13 10:59:42 +00:00
require . True ( t , okToAppend )
require . False ( t , counterReset )
} )
}
2023-05-11 19:02:02 +00:00
}
2023-01-04 10:00:06 +00:00
func TestFloatHistogramChunkAppendableGauge ( t * testing . T ) {
2024-03-22 13:36:39 +00:00
eh := & histogram . FloatHistogram {
CounterResetHint : histogram . GaugeType ,
Count : 5 ,
ZeroCount : 2 ,
Sum : 18.4 ,
ZeroThreshold : 1e-125 ,
Schema : 1 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
} ,
PositiveBuckets : [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 1 } ,
}
cbh := & histogram . FloatHistogram {
CounterResetHint : histogram . GaugeType ,
Count : 24 ,
Sum : 18.4 ,
Schema : histogram . CustomBucketsSchema ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
} ,
PositiveBuckets : [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 1 } ,
CustomValues : [ ] float64 { 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 } ,
}
setup := func ( h * histogram . FloatHistogram ) ( Chunk , * FloatHistogramAppender , int64 , * histogram . FloatHistogram ) {
2023-07-26 13:08:16 +00:00
c := Chunk ( NewFloatHistogramChunk ( ) )
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
require . Equal ( t , 0 , c . NumSamples ( ) )
ts := int64 ( 1234567890 )
2023-01-04 10:00:06 +00:00
2024-03-22 13:36:39 +00:00
chk , _ , app , err := app . AppendFloatHistogram ( nil , ts , h . Copy ( ) , false )
2023-07-26 13:08:16 +00:00
require . NoError ( t , err )
require . Nil ( t , chk )
require . Equal ( t , 1 , c . NumSamples ( ) )
require . Equal ( t , GaugeType , c . ( * FloatHistogramChunk ) . GetCounterResetHeader ( ) )
2024-03-22 13:36:39 +00:00
return c , app . ( * FloatHistogramAppender ) , ts , h
2023-01-04 10:00:06 +00:00
}
{ // Schema change.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . Schema ++
2023-07-26 13:08:16 +00:00
_ , _ , _ , _ , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-01-04 10:00:06 +00:00
require . False ( t , ok )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
{ // Zero threshold change.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . ZeroThreshold += 0.1
2023-07-26 13:08:16 +00:00
_ , _ , _ , _ , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-01-04 10:00:06 +00:00
require . False ( t , ok )
2023-07-26 13:08:16 +00:00
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
{ // New histogram that has more buckets.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . Count += 9
h2 . ZeroCount ++
h2 . Sum = 30
h2 . PositiveBuckets = [ ] float64 { 7 , 5 , 1 , 3 , 1 , 0 , 2 , 5 , 5 , 0 , 1 }
2023-07-26 13:08:16 +00:00
pI , nI , pBackwardI , nBackwardI , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-12-07 11:35:01 +00:00
require . NotEmpty ( t , pI )
require . Empty ( t , nI )
require . Empty ( t , pBackwardI )
require . Empty ( t , nBackwardI )
2023-01-04 10:00:06 +00:00
require . True ( t , ok )
2023-07-26 13:08:16 +00:00
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
{ // New histogram that has buckets missing.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 4 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
}
h2 . Count -= 4
h2 . Sum --
h2 . PositiveBuckets = [ ] float64 { 6 , 3 , 3 , 2 , 5 , 1 }
2023-07-26 13:08:16 +00:00
pI , nI , pBackwardI , nBackwardI , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-12-07 11:35:01 +00:00
require . Empty ( t , pI )
require . Empty ( t , nI )
require . NotEmpty ( t , pBackwardI )
require . Empty ( t , nBackwardI )
2023-01-04 10:00:06 +00:00
require . True ( t , ok )
2023-07-26 13:08:16 +00:00
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
{ // New histogram that has a bucket missing and new buckets.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 5 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
}
h2 . Sum = 21
h2 . PositiveBuckets = [ ] float64 { 6 , 3 , 2 , 4 , 5 , 1 }
2023-07-26 13:08:16 +00:00
pI , nI , pBackwardI , nBackwardI , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-12-07 11:35:01 +00:00
require . NotEmpty ( t , pI )
require . NotEmpty ( t , pBackwardI )
require . Empty ( t , nI )
require . Empty ( t , nBackwardI )
2023-01-04 10:00:06 +00:00
require . True ( t , ok )
2023-07-26 13:08:16 +00:00
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
{ // New histogram that has a counter reset while buckets are same.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . Sum = 23
h2 . PositiveBuckets = [ ] float64 { 6 , 2 , 3 , 2 , 4 , 5 , 1 }
2023-07-26 13:08:16 +00:00
pI , nI , pBackwardI , nBackwardI , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-12-07 11:35:01 +00:00
require . Empty ( t , pI )
require . Empty ( t , nI )
require . Empty ( t , pBackwardI )
require . Empty ( t , nBackwardI )
2023-01-04 10:00:06 +00:00
require . True ( t , ok )
2023-07-26 13:08:16 +00:00
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
{ // New histogram that has a counter reset while new buckets were added.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . Sum = 29
h2 . PositiveBuckets = [ ] float64 { 7 , 5 , 1 , 3 , 1 , 0 , 2 , 5 , 5 , 0 , 0 }
2023-07-26 13:08:16 +00:00
pI , nI , pBackwardI , nBackwardI , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-12-07 11:35:01 +00:00
require . NotEmpty ( t , pI )
require . Empty ( t , nI )
require . Empty ( t , pBackwardI )
require . Empty ( t , nBackwardI )
2023-01-04 10:00:06 +00:00
require . True ( t , ok )
2023-07-26 13:08:16 +00:00
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
{
// New histogram that has a counter reset while new buckets were
// added before the first bucket and reset on first bucket.
2024-03-22 13:36:39 +00:00
c , hApp , ts , h1 := setup ( eh )
2023-01-04 10:00:06 +00:00
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : - 3 , Length : 2 } ,
{ Offset : 1 , Length : 2 } ,
{ Offset : 2 , Length : 1 } ,
{ Offset : 3 , Length : 2 } ,
{ Offset : 3 , Length : 1 } ,
{ Offset : 1 , Length : 1 } ,
}
h2 . Sum = 26
h2 . PositiveBuckets = [ ] float64 { 1 , 2 , 5 , 3 , 3 , 2 , 4 , 5 , 1 }
2023-07-26 13:08:16 +00:00
pI , nI , pBackwardI , nBackwardI , _ , _ , ok := hApp . appendableGauge ( h2 )
2023-12-07 11:35:01 +00:00
require . NotEmpty ( t , pI )
require . Empty ( t , nI )
require . Empty ( t , pBackwardI )
require . Empty ( t , nBackwardI )
2023-01-04 10:00:06 +00:00
require . True ( t , ok )
2023-07-26 13:08:16 +00:00
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
2023-01-04 10:00:06 +00:00
}
2024-03-22 13:36:39 +00:00
{ // Custom buckets, no change.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
_ , _ , _ , _ , _ , _ , ok := hApp . appendableGauge ( h2 )
require . True ( t , ok )
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
}
{ // Custom buckets, increase in bucket counts but no change in layout.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . Count ++
h2 . PositiveBuckets = [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 2 }
_ , _ , _ , _ , _ , _ , ok := hApp . appendableGauge ( h2 )
require . True ( t , ok )
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
}
{ // Custom buckets, decrease in bucket counts but no change in layout.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . Count --
h2 . PositiveBuckets = [ ] float64 { 6 , 3 , 3 , 2 , 4 , 5 , 0 }
_ , _ , _ , _ , _ , _ , ok := hApp . appendableGauge ( h2 )
require . True ( t , ok )
assertNoNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
}
{ // Custom buckets, change only in custom bounds.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . CustomValues = [ ] float64 { 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 }
_ , _ , _ , _ , _ , _ , ok := hApp . appendableGauge ( h2 )
require . False ( t , ok )
assertNewFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
}
{ // Custom buckets, with more buckets.
c , hApp , ts , h1 := setup ( cbh )
h2 := h1 . Copy ( )
h2 . PositiveSpans = [ ] histogram . Span {
{ Offset : 0 , Length : 3 } ,
{ Offset : 1 , Length : 1 } ,
{ Offset : 1 , Length : 4 } ,
{ Offset : 3 , Length : 3 } ,
}
h2 . Count += 6
h2 . Sum = 30
// Existing histogram should get values converted from the above to:
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
h2 . PositiveBuckets = [ ] float64 { 7 , 5 , 1 , 3 , 1 , 0 , 2 , 5 , 5 , 0 , 1 } // (total 30)
posInterjections , negInterjections , pBackwardI , nBackwardI , _ , _ , ok := hApp . appendableGauge ( h2 )
require . NotEmpty ( t , posInterjections )
require . Empty ( t , negInterjections )
require . Empty ( t , pBackwardI )
require . Empty ( t , nBackwardI )
require . True ( t , ok ) // Only new buckets came in.
assertRecodedFloatHistogramChunkOnAppend ( t , c , hApp , ts + 1 , h2 , GaugeType )
}
2023-01-04 10:00:06 +00:00
}
2023-11-29 10:39:12 +00:00
func TestFloatHistogramAppendOnlyErrors ( t * testing . T ) {
t . Run ( "schema change error" , func ( t * testing . T ) {
c := Chunk ( NewFloatHistogramChunk ( ) )
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
h := tsdbutil . GenerateTestFloatHistogram ( 0 )
var isRecoded bool
c , isRecoded , app , err = app . AppendFloatHistogram ( nil , 1 , h , true )
require . Nil ( t , c )
require . False ( t , isRecoded )
require . NoError ( t , err )
// Add erroring histogram.
h2 := h . Copy ( )
h2 . Schema ++
c , isRecoded , _ , err = app . AppendFloatHistogram ( nil , 2 , h2 , true )
require . Nil ( t , c )
require . False ( t , isRecoded )
require . EqualError ( t , err , "float histogram schema change" )
} )
t . Run ( "counter reset error" , func ( t * testing . T ) {
c := Chunk ( NewFloatHistogramChunk ( ) )
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
h := tsdbutil . GenerateTestFloatHistogram ( 0 )
var isRecoded bool
c , isRecoded , app , err = app . AppendFloatHistogram ( nil , 1 , h , true )
require . Nil ( t , c )
require . False ( t , isRecoded )
require . NoError ( t , err )
// Add erroring histogram.
h2 := h . Copy ( )
h2 . CounterResetHint = histogram . CounterReset
c , isRecoded , _ , err = app . AppendFloatHistogram ( nil , 2 , h2 , true )
require . Nil ( t , c )
require . False ( t , isRecoded )
require . EqualError ( t , err , "float histogram counter reset" )
} )
2024-03-22 13:36:39 +00:00
t . Run ( "counter reset error with custom buckets" , func ( t * testing . T ) {
c := Chunk ( NewFloatHistogramChunk ( ) )
// Create fresh appender and add the first histogram.
app , err := c . Appender ( )
require . NoError ( t , err )
h := tsdbutil . GenerateTestCustomBucketsFloatHistogram ( 0 )
var isRecoded bool
c , isRecoded , app , err = app . AppendFloatHistogram ( nil , 1 , h , true )
require . Nil ( t , c )
require . False ( t , isRecoded )
require . NoError ( t , err )
// Add erroring histogram.
h2 := h . Copy ( )
h2 . CustomValues = [ ] float64 { 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 }
c , isRecoded , _ , err = app . AppendFloatHistogram ( nil , 2 , h2 , true )
require . Nil ( t , c )
require . False ( t , isRecoded )
require . EqualError ( t , err , "float histogram counter reset" )
} )
2023-11-29 10:39:12 +00:00
}
2024-09-04 10:07:16 +00:00
func TestFloatHistogramUniqueSpansAfterNext ( t * testing . T ) {
// Create two histograms with the same schema and spans.
h1 := & histogram . FloatHistogram {
Schema : 1 ,
ZeroThreshold : 1e-100 ,
Count : 10 ,
ZeroCount : 2 ,
Sum : 15.0 ,
PositiveSpans : [ ] histogram . Span {
{ Offset : 0 , Length : 2 } ,
{ Offset : 1 , Length : 2 } ,
} ,
PositiveBuckets : [ ] float64 { 1 , 2 , 3 , 4 } ,
NegativeSpans : [ ] histogram . Span {
{ Offset : 1 , Length : 1 } ,
} ,
NegativeBuckets : [ ] float64 { 2 } ,
}
h2 := h1 . Copy ( )
// Create a chunk and append both histograms.
c := NewFloatHistogramChunk ( )
app , err := c . Appender ( )
require . NoError ( t , err )
_ , _ , _ , err = app . AppendFloatHistogram ( nil , 0 , h1 , false )
require . NoError ( t , err )
_ , _ , _ , err = app . AppendFloatHistogram ( nil , 1 , h2 , false )
require . NoError ( t , err )
// Create an iterator and advance to the first histogram.
it := c . Iterator ( nil )
require . Equal ( t , ValFloatHistogram , it . Next ( ) )
_ , rh1 := it . AtFloatHistogram ( nil )
// Advance to the second histogram and retrieve it.
require . Equal ( t , ValFloatHistogram , it . Next ( ) )
_ , rh2 := it . AtFloatHistogram ( nil )
require . Equal ( t , rh1 . PositiveSpans , h1 . PositiveSpans , "Returned positive spans are as expected" )
require . Equal ( t , rh1 . NegativeSpans , h1 . NegativeSpans , "Returned negative spans are as expected" )
require . Equal ( t , rh2 . PositiveSpans , h1 . PositiveSpans , "Returned positive spans are as expected" )
require . Equal ( t , rh2 . NegativeSpans , h1 . NegativeSpans , "Returned negative spans are as expected" )
// Check that the spans for h1 and h2 are unique slices.
require . NotSame ( t , & rh1 . PositiveSpans [ 0 ] , & rh2 . PositiveSpans [ 0 ] , "PositiveSpans should be unique between histograms" )
require . NotSame ( t , & rh1 . NegativeSpans [ 0 ] , & rh2 . NegativeSpans [ 0 ] , "NegativeSpans should be unique between histograms" )
}