@ -18,6 +18,7 @@ import (
"math"
"math/rand"
"os"
"path"
"path/filepath"
"sort"
"testing"
@ -296,94 +297,158 @@ func TestHeadDeleteSeriesWithoutSamples(t *testing.T) {
}
func TestHeadDeleteSimple ( t * testing . T ) {
numSamples := int64 ( 10 )
head , err := NewHead ( nil , nil , nil , 1000 )
testutil . Ok ( t , err )
defer head . Close ( )
app := head . Appender ( )
smpls := make ( [ ] float64 , numSamples )
for i := int64 ( 0 ) ; i < numSamples ; i ++ {
smpls [ i ] = rand . Float64 ( )
app . Add ( labels . Labels { { "a" , "b" } } , i , smpls [ i ] )
buildSmpls := func ( s [ ] int64 ) [ ] sample {
ss := make ( [ ] sample , 0 , len ( s ) )
for _ , t := range s {
ss = append ( ss , sample { t : t , v : float64 ( t ) } )
}
return ss
}
smplsAll := buildSmpls ( [ ] int64 { 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 } )
lblDefault := labels . Label { "a" , "b" }
testutil . Ok ( t , app . Commit ( ) )
cases := [ ] struct {
intervals Intervals
remaint [ ] int64
dranges Intervals
smplsExp [ ] sample
} {
{
intervals: Intervals { { 0 , 3 } } ,
remaint: [ ] int64 { 4 , 5 , 6 , 7 , 8 , 9 } ,
dranges: Intervals { { 0 , 3 } } ,
smplsExp: buildSmpls ( [ ] int64 { 4 , 5 , 6 , 7 , 8 , 9 } ) ,
} ,
{
intervals: Intervals { { 1 , 3 } } ,
remaint: [ ] int64 { 0 , 4 , 5 , 6 , 7 , 8 , 9 } ,
dranges: Intervals { { 1 , 3 } } ,
smplsExp: buildSmpls ( [ ] int64 { 0 , 4 , 5 , 6 , 7 , 8 , 9 } ) ,
} ,
{
intervals: Intervals { { 1 , 3 } , { 4 , 7 } } ,
remaint: [ ] int64 { 0 , 8 , 9 } ,
dranges: Intervals { { 1 , 3 } , { 4 , 7 } } ,
smplsExp: buildSmpls ( [ ] int64 { 0 , 8 , 9 } ) ,
} ,
{
intervals: Intervals { { 1 , 3 } , { 4 , 700 } } ,
remaint: [ ] int64 { 0 } ,
dranges: Intervals { { 1 , 3 } , { 4 , 700 } } ,
smplsExp: buildSmpls ( [ ] int64 { 0 } ) ,
} ,
{
intervals: Intervals { { 0 , 9 } } ,
remaint: [ ] int64 { } ,
{ // This case is to ensure that labels and symbols are deleted.
dranges: Intervals { { 0 , 9 } } ,
smplsExp: buildSmpls ( [ ] int64 { } ) ,
} ,
}
Outer :
for _ , c := range cases {
// Reset the tombstones.
head . tombstones = newMemTombstones ( )
dir , err := ioutil . TempDir ( "" , "test_wal_reload" )
testutil . Ok ( t , err )
defer os . RemoveAll ( dir )
w , err := wal . New ( nil , nil , path . Join ( dir , "wal" ) )
testutil . Ok ( t , err )
head , err := NewHead ( nil , nil , w , 1000 )
testutil . Ok ( t , err )
app := head . Appender ( )
for _ , smpl := range smplsAll {
_ , err = app . Add ( labels . Labels { lblDefault } , smpl . t , smpl . v )
testutil . Ok ( t , err )
}
testutil . Ok ( t , app . Commit ( ) )
// Delete the ranges.
for _ , r := range c . intervals {
testutil . Ok ( t , head . Delete ( r . Mint , r . Maxt , labels . NewEqualMatcher ( "a" , "b" ) ) )
for _ , r := range c . drange s {
testutil . Ok ( t , head . Delete ( r . Mint , r . Maxt , labels . NewEqualMatcher ( lblDefault . Name , lblDefault . Value ) ) )
}
// Compare the result.
q , err := NewBlockQuerier ( head , head . MinTime ( ) , head . MaxTime ( ) )
// Compare the samples for both heads - before and after the reload .
reloadedW, err := wal . New ( nil , nil , w . Dir ( ) ) // Use a new wal to ensure deleted samples are gone even after a reload.
testutil . Ok ( t , err )
res , err := q . Select ( labels . NewEqualMatcher ( "a" , "b" ) )
re loadedHead, err := NewHead ( nil , nil , reloadedW , 1000 )
testutil . Ok ( t , err )
testutil . Ok ( t , reloadedHead . Init ( 0 ) )
for _ , h := range [ ] * Head { head , reloadedHead } {
indexr , err := h . Index ( )
testutil . Ok ( t , err )
// Use an emptyTombstoneReader explicitly to get all the samples.
css , err := LookupChunkSeries ( indexr , emptyTombstoneReader , labels . NewEqualMatcher ( lblDefault . Name , lblDefault . Value ) )
testutil . Ok ( t , err )
expSamples := make ( [ ] tsdbutil . Sample , 0 , len ( c . remaint ) )
for _ , ts := range c . remaint {
expSamples = append ( expSamples , sample { ts , smpls [ ts ] } )
// Getting the actual samples.
actSamples := make ( [ ] sample , 0 )
for css . Next ( ) {
lblsAct , chkMetas , intv := css . At ( )
testutil . Equals ( t , labels . Labels { lblDefault } , lblsAct )
testutil . Equals ( t , 0 , len ( intv ) )
chunkr , err := h . Chunks ( )
testutil . Ok ( t , err )
for _ , meta := range chkMetas {
chk , err := chunkr . Chunk ( meta . Ref )
testutil . Ok ( t , err )
ii := chk . Iterator ( )
for ii . Next ( ) {
t , v := ii . At ( )
actSamples = append ( actSamples , sample { t : t , v : v } )
}
}
}
testutil . Ok ( t , css . Err ( ) )
testutil . Equals ( t , c . smplsExp , actSamples )
}
expss := newMockSeriesSet ( [ ] Series {
newSeries ( map [ string ] string { "a" : "b" } , expSamples ) ,
// Compare the query results for both heads - before and after the reload.
expSeriesSet := newMockSeriesSet ( [ ] Series {
newSeries ( map [ string ] string { lblDefault . Name : lblDefault . Value } , func ( ) [ ] tsdbutil . Sample {
ss := make ( [ ] tsdbutil . Sample , 0 , len ( c . smplsExp ) )
for _ , s := range c . smplsExp {
ss = append ( ss , s )
}
return ss
} ( ) ,
) ,
} )
for _ , h := range [ ] * Head { head , reloadedHead } {
q , err := NewBlockQuerier ( h , h . MinTime ( ) , h . MaxTime ( ) )
testutil . Ok ( t , err )
actSeriesSet , err := q . Select ( labels . NewEqualMatcher ( lblDefault . Name , lblDefault . Value ) )
testutil . Ok ( t , err )
if len ( expSamples ) == 0 {
testutil . Assert ( t , res . Next ( ) == false , "" )
continue
}
lns , err := q . LabelNames ( )
testutil . Ok ( t , err )
lvs , err := q . LabelValues ( lblDefault . Name )
testutil . Ok ( t , err )
// When all samples are deleted we expect that no labels should exist either.
if len ( c . smplsExp ) == 0 {
testutil . Equals ( t , 0 , len ( lns ) )
testutil . Equals ( t , 0 , len ( lvs ) )
testutil . Assert ( t , actSeriesSet . Next ( ) == false , "" )
testutil . Ok ( t , h . Close ( ) )
continue
} else {
testutil . Equals ( t , 1 , len ( lns ) )
testutil . Equals ( t , 1 , len ( lvs ) )
testutil . Equals ( t , lblDefault . Name , lns [ 0 ] )
testutil . Equals ( t , lblDefault . Value , lvs [ 0 ] )
}
for {
eok , rok := expss . Next ( ) , res . Next ( )
testutil . Equals ( t , eok , rok )
for {
eok , rok := exp SeriesSet . Next ( ) , actSe ri esSet . Next ( )
testutil . Equals ( t , eok , rok )
if ! eok {
continue Outer
}
sexp := expss . At ( )
sres := res . At ( )
if ! eok {
testutil . Ok ( t , h . Close ( ) )
continue Outer
}
expSeries := expSeriesSet . At ( )
actSeries := actSeriesSet . At ( )
testutil . Equals ( t , sexp . Labels ( ) , sres . Labels ( ) )
testutil . Equals ( t , Series . Labels ( ) , actSeri es. Labels ( ) )
smplExp , errExp := expandSeriesIterator ( s exp. Iterator ( ) )
smplRes , errRes := expandSeriesIterator ( sr es. Iterator ( ) )
smplExp , errExp := expandSeriesIterator ( Series . Iterator ( ) )
smplRes , errRes := expandSeriesIterator ( actSeri es. Iterator ( ) )
testutil . Equals ( t , errExp , errRes )
testutil . Equals ( t , smplExp , smplRes )
testutil . Equals ( t , errExp , errRes )
testutil . Equals ( t , smplExp , smplRes )
}
}
}
}
@ -524,8 +589,6 @@ func TestDelete_e2e(t *testing.T) {
// TODO: Add Regexp Matchers.
}
for _ , del := range dels {
// Reset the deletes everytime.
hb . tombstones = newMemTombstones ( )
for _ , r := range del . drange {
testutil . Ok ( t , hb . Delete ( r . Mint , r . Maxt , del . ms ... ) )
}
@ -585,7 +648,6 @@ func TestDelete_e2e(t *testing.T) {
}
}
}
return
}
func boundedSamples ( full [ ] sample , mint , maxt int64 ) [ ] sample {
@ -946,4 +1008,5 @@ func TestWalRepair(t *testing.T) {
testutil . Equals ( t , test . expRecs , actRec , "Wrong number of intact records" )
} )
}
}
Krasi Georgiev
6 years ago