Browse Source

use test utils in wal_test

pull/5805/head
Callum Styan 7 years ago
parent
commit
6c8bbf3f61
  1. 129
      wal_test.go

129
wal_test.go

@ -22,47 +22,46 @@ import (
"github.com/go-kit/kit/log" "github.com/go-kit/kit/log"
"github.com/prometheus/tsdb/fileutil" "github.com/prometheus/tsdb/fileutil"
"github.com/stretchr/testify/require"
) )
func TestSegmentWAL_cut(t *testing.T) { func TestSegmentWAL_cut(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "test_wal_cut") tmpdir, err := ioutil.TempDir("", "test_wal_cut")
require.NoError(t, err) Ok(t, err)
defer os.RemoveAll(tmpdir) defer os.RemoveAll(tmpdir)
// This calls cut() implicitly the first time without a previous tail. // This calls cut() implicitly the first time without a previous tail.
w, err := OpenSegmentWAL(tmpdir, nil, 0, nil) w, err := OpenSegmentWAL(tmpdir, nil, 0, nil)
require.NoError(t, err) Ok(t, err)
require.NoError(t, w.write(WALEntrySeries, 1, []byte("Hello World!!"))) Ok(t, w.write(WALEntrySeries, 1, []byte("Hello World!!")))
require.NoError(t, w.cut(), "cut failed") Ok(t, w.cut())
// Cutting creates a new file. // Cutting creates a new file.
require.Equal(t, 2, len(w.files)) Equals(t, 2, len(w.files))
require.NoError(t, w.write(WALEntrySeries, 1, []byte("Hello World!!"))) Ok(t, w.write(WALEntrySeries, 1, []byte("Hello World!!")))
require.NoError(t, w.Close()) Ok(t, w.Close())
for _, of := range w.files { for _, of := range w.files {
f, err := os.Open(of.Name()) f, err := os.Open(of.Name())
require.NoError(t, err) Ok(t, err)
// Verify header data. // Verify header data.
metab := make([]byte, 8) metab := make([]byte, 8)
_, err = f.Read(metab) _, err = f.Read(metab)
require.NoError(t, err, "read meta data %s", f.Name()) Ok(t, err)
require.Equal(t, WALMagic, binary.BigEndian.Uint32(metab[:4]), "verify magic") Equals(t, WALMagic, binary.BigEndian.Uint32(metab[:4]))
require.Equal(t, WALFormatDefault, metab[4], "verify format flag") Equals(t, WALFormatDefault, metab[4])
// We cannot actually check for correct pre-allocation as it is // We cannot actually check for correct pre-allocation as it is
// optional per filesystem and handled transparently. // optional per filesystem and handled transparently.
et, flag, b, err := newWALReader(nil, nil).entry(f) et, flag, b, err := newWALReader(nil, nil).entry(f)
require.NoError(t, err) Ok(t, err)
require.Equal(t, WALEntrySeries, et) Equals(t, WALEntrySeries, et)
require.Equal(t, flag, byte(walSeriesSimple)) Equals(t, flag, byte(walSeriesSimple))
require.Equal(t, []byte("Hello World!!"), b) Equals(t, []byte("Hello World!!"), b)
} }
} }
@ -72,14 +71,14 @@ func TestSegmentWAL_Truncate(t *testing.T) {
batch = 100 batch = 100
) )
series, err := readPrometheusLabels("testdata/20k.series", numMetrics) series, err := readPrometheusLabels("testdata/20k.series", numMetrics)
require.NoError(t, err) Ok(t, err)
dir, err := ioutil.TempDir("", "test_wal_log_truncate") dir, err := ioutil.TempDir("", "test_wal_log_truncate")
require.NoError(t, err) Ok(t, err)
// defer os.RemoveAll(dir) // defer os.RemoveAll(dir)
w, err := OpenSegmentWAL(dir, nil, 0, nil) w, err := OpenSegmentWAL(dir, nil, 0, nil)
require.NoError(t, err) Ok(t, err)
w.segmentSize = 10000 w.segmentSize = 10000
for i := 0; i < numMetrics; i += batch { for i := 0; i < numMetrics; i += batch {
@ -89,7 +88,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
rs = append(rs, RefSeries{Labels: s, Ref: uint64(i+j) + 1}) rs = append(rs, RefSeries{Labels: s, Ref: uint64(i+j) + 1})
} }
err := w.LogSeries(rs) err := w.LogSeries(rs)
require.NoError(t, err) Ok(t, err)
} }
// We mark the 2nd half of the files with a min timestamp that should discard // We mark the 2nd half of the files with a min timestamp that should discard
@ -111,7 +110,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
} }
err = w.Truncate(1000, keepf) err = w.Truncate(1000, keepf)
require.NoError(t, err) Ok(t, err)
var expected []RefSeries var expected []RefSeries
@ -124,12 +123,12 @@ func TestSegmentWAL_Truncate(t *testing.T) {
// Call Truncate once again to see whether we can read the written file without // Call Truncate once again to see whether we can read the written file without
// creating a new WAL. // creating a new WAL.
err = w.Truncate(1000, keepf) err = w.Truncate(1000, keepf)
require.NoError(t, err) Ok(t, err)
require.NoError(t, w.Close()) Ok(t, w.Close())
// The same again with a new WAL. // The same again with a new WAL.
w, err = OpenSegmentWAL(dir, nil, 0, nil) w, err = OpenSegmentWAL(dir, nil, 0, nil)
require.NoError(t, err) Ok(t, err)
var readSeries []RefSeries var readSeries []RefSeries
r := w.Reader() r := w.Reader()
@ -138,7 +137,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
readSeries = append(readSeries, s...) readSeries = append(readSeries, s...)
}, nil, nil) }, nil, nil)
require.Equal(t, expected, readSeries) Equals(t, expected, readSeries)
} }
// Symmetrical test of reading and writing to the WAL via its main interface. // Symmetrical test of reading and writing to the WAL via its main interface.
@ -151,10 +150,10 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
// Generate testing data. It does not make semantical sense but // Generate testing data. It does not make semantical sense but
// for the purpose of this test. // for the purpose of this test.
series, err := readPrometheusLabels("testdata/20k.series", numMetrics) series, err := readPrometheusLabels("testdata/20k.series", numMetrics)
require.NoError(t, err) Ok(t, err)
dir, err := ioutil.TempDir("", "test_wal_log_restore") dir, err := ioutil.TempDir("", "test_wal_log_restore")
require.NoError(t, err) Ok(t, err)
defer os.RemoveAll(dir) defer os.RemoveAll(dir)
var ( var (
@ -168,7 +167,7 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
// write more data to it, close it. // write more data to it, close it.
for k := 0; k < numMetrics; k += numMetrics / iterations { for k := 0; k < numMetrics; k += numMetrics / iterations {
w, err := OpenSegmentWAL(dir, nil, 0, nil) w, err := OpenSegmentWAL(dir, nil, 0, nil)
require.NoError(t, err) Ok(t, err)
// Set smaller segment size so we can actually write several files. // Set smaller segment size so we can actually write several files.
w.segmentSize = 1000 * 1000 w.segmentSize = 1000 * 1000
@ -204,11 +203,11 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
} }
} }
require.NoError(t, r.Read(serf, smplf, delf)) Ok(t, r.Read(serf, smplf, delf))
require.Equal(t, recordedSamples, resultSamples) Equals(t, recordedSamples, resultSamples)
require.Equal(t, recordedSeries, resultSeries) Equals(t, recordedSeries, resultSeries)
require.Equal(t, recordedDeletes, resultDeletes) Equals(t, recordedDeletes, resultDeletes)
series := series[k : k+(numMetrics/iterations)] series := series[k : k+(numMetrics/iterations)]
@ -239,9 +238,9 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
}) })
} }
require.NoError(t, w.LogSeries(series)) Ok(t, w.LogSeries(series))
require.NoError(t, w.LogSamples(samples)) Ok(t, w.LogSamples(samples))
require.NoError(t, w.LogDeletes(stones)) Ok(t, w.LogDeletes(stones))
if len(lbls) > 0 { if len(lbls) > 0 {
recordedSeries = append(recordedSeries, series) recordedSeries = append(recordedSeries, series)
@ -255,7 +254,7 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
} }
} }
require.NoError(t, w.Close()) Ok(t, w.Close())
} }
} }
@ -300,56 +299,56 @@ func TestWALRestoreCorrupted(t *testing.T) {
name: "truncate_checksum", name: "truncate_checksum",
f: func(t *testing.T, w *SegmentWAL) { f: func(t *testing.T, w *SegmentWAL) {
f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666) f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666)
require.NoError(t, err) Ok(t, err)
defer f.Close() defer f.Close()
off, err := f.Seek(0, os.SEEK_END) off, err := f.Seek(0, os.SEEK_END)
require.NoError(t, err) Ok(t, err)
require.NoError(t, f.Truncate(off-1)) Ok(t, f.Truncate(off-1))
}, },
}, },
{ {
name: "truncate_body", name: "truncate_body",
f: func(t *testing.T, w *SegmentWAL) { f: func(t *testing.T, w *SegmentWAL) {
f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666) f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666)
require.NoError(t, err) Ok(t, err)
defer f.Close() defer f.Close()
off, err := f.Seek(0, os.SEEK_END) off, err := f.Seek(0, os.SEEK_END)
require.NoError(t, err) Ok(t, err)
require.NoError(t, f.Truncate(off-8)) Ok(t, f.Truncate(off-8))
}, },
}, },
{ {
name: "body_content", name: "body_content",
f: func(t *testing.T, w *SegmentWAL) { f: func(t *testing.T, w *SegmentWAL) {
f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666) f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666)
require.NoError(t, err) Ok(t, err)
defer f.Close() defer f.Close()
off, err := f.Seek(0, os.SEEK_END) off, err := f.Seek(0, os.SEEK_END)
require.NoError(t, err) Ok(t, err)
// Write junk before checksum starts. // Write junk before checksum starts.
_, err = f.WriteAt([]byte{1, 2, 3, 4}, off-8) _, err = f.WriteAt([]byte{1, 2, 3, 4}, off-8)
require.NoError(t, err) Ok(t, err)
}, },
}, },
{ {
name: "checksum", name: "checksum",
f: func(t *testing.T, w *SegmentWAL) { f: func(t *testing.T, w *SegmentWAL) {
f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666) f, err := os.OpenFile(w.files[0].Name(), os.O_WRONLY, 0666)
require.NoError(t, err) Ok(t, err)
defer f.Close() defer f.Close()
off, err := f.Seek(0, os.SEEK_END) off, err := f.Seek(0, os.SEEK_END)
require.NoError(t, err) Ok(t, err)
// Write junk into checksum // Write junk into checksum
_, err = f.WriteAt([]byte{1, 2, 3, 4}, off-4) _, err = f.WriteAt([]byte{1, 2, 3, 4}, off-4)
require.NoError(t, err) Ok(t, err)
}, },
}, },
} }
@ -358,21 +357,21 @@ func TestWALRestoreCorrupted(t *testing.T) {
// Generate testing data. It does not make semantical sense but // Generate testing data. It does not make semantical sense but
// for the purpose of this test. // for the purpose of this test.
dir, err := ioutil.TempDir("", "test_corrupted") dir, err := ioutil.TempDir("", "test_corrupted")
require.NoError(t, err) Ok(t, err)
defer os.RemoveAll(dir) defer os.RemoveAll(dir)
w, err := OpenSegmentWAL(dir, nil, 0, nil) w, err := OpenSegmentWAL(dir, nil, 0, nil)
require.NoError(t, err) Ok(t, err)
require.NoError(t, w.LogSamples([]RefSample{{T: 1, V: 2}})) Ok(t, w.LogSamples([]RefSample{{T: 1, V: 2}}))
require.NoError(t, w.LogSamples([]RefSample{{T: 2, V: 3}})) Ok(t, w.LogSamples([]RefSample{{T: 2, V: 3}}))
require.NoError(t, w.cut()) Ok(t, w.cut())
require.NoError(t, w.LogSamples([]RefSample{{T: 3, V: 4}})) Ok(t, w.LogSamples([]RefSample{{T: 3, V: 4}}))
require.NoError(t, w.LogSamples([]RefSample{{T: 5, V: 6}})) Ok(t, w.LogSamples([]RefSample{{T: 5, V: 6}}))
require.NoError(t, w.Close()) Ok(t, w.Close())
// cut() truncates and fsyncs the first segment async. If it happens after // cut() truncates and fsyncs the first segment async. If it happens after
// the corruption we apply below, the corruption will be overwritten again. // the corruption we apply below, the corruption will be overwritten again.
@ -387,39 +386,39 @@ func TestWALRestoreCorrupted(t *testing.T) {
logger := log.NewLogfmtLogger(os.Stderr) logger := log.NewLogfmtLogger(os.Stderr)
w2, err := OpenSegmentWAL(dir, logger, 0, nil) w2, err := OpenSegmentWAL(dir, logger, 0, nil)
require.NoError(t, err) Ok(t, err)
r := w2.Reader() r := w2.Reader()
serf := func(l []RefSeries) { serf := func(l []RefSeries) {
require.Equal(t, 0, len(l)) Equals(t, 0, len(l))
} }
// Weird hack to check order of reads. // Weird hack to check order of reads.
i := 0 i := 0
samplf := func(s []RefSample) { samplf := func(s []RefSample) {
if i == 0 { if i == 0 {
require.Equal(t, []RefSample{{T: 1, V: 2}}, s) Equals(t, []RefSample{{T: 1, V: 2}}, s)
i++ i++
} else { } else {
require.Equal(t, []RefSample{{T: 99, V: 100}}, s) Equals(t, []RefSample{{T: 99, V: 100}}, s)
} }
} }
require.NoError(t, r.Read(serf, samplf, nil)) Ok(t, r.Read(serf, samplf, nil))
require.NoError(t, w2.LogSamples([]RefSample{{T: 99, V: 100}})) Ok(t, w2.LogSamples([]RefSample{{T: 99, V: 100}}))
require.NoError(t, w2.Close()) Ok(t, w2.Close())
// We should see the first valid entry and the new one, everything after // We should see the first valid entry and the new one, everything after
// is truncated. // is truncated.
w3, err := OpenSegmentWAL(dir, logger, 0, nil) w3, err := OpenSegmentWAL(dir, logger, 0, nil)
require.NoError(t, err) Ok(t, err)
r = w3.Reader() r = w3.Reader()
i = 0 i = 0
require.NoError(t, r.Read(serf, samplf, nil)) Ok(t, r.Read(serf, samplf, nil))
}) })
} }
} }

Loading…
Cancel
Save