From adb6be73dd2dd7a3063df45149bde6ac901ea7c8 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Thu, 21 Dec 2017 11:55:58 +0100 Subject: [PATCH] move label read function int labels pkg --- block_test.go | 4 +-- head_test.go | 55 +++-------------------------------------- index/index_test.go | 57 ++++++++----------------------------------- labels/labels.go | 49 +++++++++++++++++++++++++++++++++++++ labels/labels_test.go | 42 +------------------------------ querier_test.go | 4 +-- wal_test.go | 5 ++-- 7 files changed, 71 insertions(+), 145 deletions(-) diff --git a/block_test.go b/block_test.go index 60d9fe54f..690cd07ad 100644 --- a/block_test.go +++ b/block_test.go @@ -45,8 +45,8 @@ func createEmptyBlock(t *testing.T, dir string) *Block { testutil.Ok(t, writeMetaFile(dir, &BlockMeta{})) ir, err := index.NewWriter(filepath.Join(dir, indexFilename)) - Ok(t, err) - Ok(t, ir.Close()) + testutil.Ok(t, err) + testutil.Ok(t, ir.Close()) testutil.Ok(t, os.MkdirAll(chunkDir(dir), 0777)) diff --git a/head_test.go b/head_test.go index b88f34a1e..36a97542b 100644 --- a/head_test.go +++ b/head_test.go @@ -14,22 +14,18 @@ package tsdb import ( - "bufio" "math/rand" - "os" - "sort" - "strings" "testing" - "github.com/pkg/errors" "github.com/prometheus/tsdb/chunkenc" + "github.com/prometheus/tsdb/chunks" "github.com/prometheus/tsdb/index" "github.com/prometheus/tsdb/labels" "github.com/prometheus/tsdb/testutil" ) func BenchmarkCreateSeries(b *testing.B) { - lbls, err := readPrometheusLabels("testdata/all.series", b.N) + lbls, err := labels.ReadLabels("testdata/all.series", b.N) testutil.Ok(b, err) h, err := NewHead(nil, nil, nil, 10000) @@ -46,49 +42,6 @@ func BenchmarkCreateSeries(b *testing.B) { } } -func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) { - f, err := os.Open(fn) - if err != nil { - return nil, err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - - var mets []labels.Labels - hashes := map[uint64]struct{}{} - i := 0 - - for scanner.Scan() && i < n { - m := make(labels.Labels, 0, 10) - - r := strings.NewReplacer("\"", "", "{", "", "}", "") - s := r.Replace(scanner.Text()) - - labelChunks := strings.Split(s, ",") - for _, labelChunk := range labelChunks { - split := strings.Split(labelChunk, ":") - m = append(m, labels.Label{Name: split[0], Value: split[1]}) - } - // Order of the k/v labels matters, don't assume we'll always receive them already sorted. - sort.Sort(m) - h := m.Hash() - if _, ok := hashes[h]; ok { - continue - } - mets = append(mets, m) - hashes[h] = struct{}{} - i++ - } - if err != nil { - return nil, err - } - if i != n { - return mets, errors.Errorf("requested %d metrics but found %d", n, i) - } - return mets, nil -} - type memoryWAL struct { nopWAL entries []interface{} @@ -727,7 +680,7 @@ func TestGCChunkAccess(t *testing.T) { idx := h.indexRange(0, 1500) var ( lset labels.Labels - chunks []ChunkMeta + chunks []chunks.Meta ) testutil.Ok(t, idx.Series(1, &lset, &chunks)) @@ -767,7 +720,7 @@ func TestGCSeriesAccess(t *testing.T) { idx := h.indexRange(0, 2000) var ( lset labels.Labels - chunks []ChunkMeta + chunks []chunks.Meta ) testutil.Ok(t, idx.Series(1, &lset, &chunks)) diff --git a/index/index_test.go b/index/index_test.go index f4ba813f3..83b6ef657 100644 --- a/index/index_test.go +++ b/index/index_test.go @@ -20,11 +20,8 @@ import ( "path/filepath" "sort" "testing" - "unsafe" "github.com/pkg/errors" - promlabels "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/tsdb/chunkenc" "github.com/prometheus/tsdb/chunks" "github.com/prometheus/tsdb/labels" @@ -242,7 +239,7 @@ func TestPersistence_index_e2e(t *testing.T) { testutil.Ok(t, err) defer os.RemoveAll(dir) - lbls, err := readPrometheusLabels("testdata/20kseries.json", 20000) + lbls, err := labels.ReadLabels("../testdata/20kseries.json", 20000) testutil.Ok(t, err) // Sort labels as the index writer expects series in sorted order. @@ -307,7 +304,11 @@ func TestPersistence_index_e2e(t *testing.T) { } for k, v := range values { - vals := v.slice() + var vals []string + for e := range v { + vals = append(vals, e) + } + sort.Strings(vals) testutil.Ok(t, iw.WriteLabelIndex([]string{k}, vals)) testutil.Ok(t, mi.WriteLabelIndex([]string{k}, vals)) @@ -322,15 +323,15 @@ func TestPersistence_index_e2e(t *testing.T) { mi.WritePostings("", "", newListPostings(all)) for l := range postings.m { - err = iw.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value)) + err = iw.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value)) testutil.Ok(t, err) - mi.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value)) + mi.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value)) } err = iw.Close() testutil.Ok(t, err) - ir, err := NewFileIndexReader(filepath.Join(dir, "index")) + ir, err := NewFileReader(filepath.Join(dir, "index")) testutil.Ok(t, err) for p := range mi.postings { @@ -359,7 +360,7 @@ func TestPersistence_index_e2e(t *testing.T) { } for k, v := range mi.labelIndex { - tplsExp, err := newStringTuples(v, 1) + tplsExp, err := NewStringTuples(v, 1) testutil.Ok(t, err) tplsRes, err := ir.LabelValues(k) @@ -379,41 +380,3 @@ func TestPersistence_index_e2e(t *testing.T) { testutil.Ok(t, ir.Close()) } - -func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) { - f, err := os.Open(fn) - if err != nil { - return nil, err - } - defer f.Close() - - b, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - p := textparse.New(b) - i := 0 - var mets []labels.Labels - hashes := map[uint64]struct{}{} - - for p.Next() && i < n { - m := make(labels.Labels, 0, 10) - p.Metric((*promlabels.Labels)(unsafe.Pointer(&m))) - - h := m.Hash() - if _, ok := hashes[h]; ok { - continue - } - mets = append(mets, m) - hashes[h] = struct{}{} - i++ - } - if err := p.Err(); err != nil { - return nil, err - } - if i != n { - return mets, errors.Errorf("requested %d metrics but found %d", n, i) - } - return mets, nil -} diff --git a/labels/labels.go b/labels/labels.go index 2ee42360e..d76ba0d08 100644 --- a/labels/labels.go +++ b/labels/labels.go @@ -14,12 +14,15 @@ package labels import ( + "bufio" "bytes" + "os" "sort" "strconv" "strings" "github.com/cespare/xxhash" + "github.com/pkg/errors" ) const sep = '\xff' @@ -161,3 +164,49 @@ type Slice []Labels func (s Slice) Len() int { return len(s) } func (s Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s Slice) Less(i, j int) bool { return Compare(s[i], s[j]) < 0 } + +// ReadLabels reads up to n label sets in a JSON formatted file fn. It is mostly useful +// to load testing data. +func ReadLabels(fn string, n int) ([]Labels, error) { + f, err := os.Open(fn) + if err != nil { + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + + var mets []Labels + hashes := map[uint64]struct{}{} + i := 0 + + for scanner.Scan() && i < n { + m := make(Labels, 0, 10) + + r := strings.NewReplacer("\"", "", "{", "", "}", "") + s := r.Replace(scanner.Text()) + + labelChunks := strings.Split(s, ",") + for _, labelChunk := range labelChunks { + split := strings.Split(labelChunk, ":") + m = append(m, Label{Name: split[0], Value: split[1]}) + } + // Order of the k/v labels matters, don't assume we'll always receive them already sorted. + sort.Sort(m) + + h := m.Hash() + if _, ok := hashes[h]; ok { + continue + } + mets = append(mets, m) + hashes[h] = struct{}{} + i++ + } + if err != nil { + return nil, err + } + if i != n { + return mets, errors.Errorf("requested %d metrics but found %d", n, i) + } + return mets, nil +} diff --git a/labels/labels_test.go b/labels/labels_test.go index e57c5c29f..709568288 100644 --- a/labels/labels_test.go +++ b/labels/labels_test.go @@ -14,12 +14,9 @@ package labels import ( - "bufio" "fmt" "math/rand" - "os" "sort" - "strings" "testing" "github.com/prometheus/tsdb/testutil" @@ -90,7 +87,7 @@ func TestCompareAndEquals(t *testing.T) { } func BenchmarkSliceSort(b *testing.B) { - lbls, err := readPrometheusLabels("../testdata/1m.series", 900000) + lbls, err := ReadLabels("../testdata/1m.series", 900000) testutil.Ok(b, err) for len(lbls) < 20e6 { @@ -119,43 +116,6 @@ func BenchmarkSliceSort(b *testing.B) { } } -func readPrometheusLabels(fn string, n int) ([]Labels, error) { - f, err := os.Open(fn) - if err != nil { - return nil, err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - - var mets []Labels - hashes := map[uint64]struct{}{} - i := 0 - - for scanner.Scan() && i < n { - m := make(Labels, 0, 10) - - r := strings.NewReplacer("\"", "", "{", "", "}", "") - s := r.Replace(scanner.Text()) - - labelChunks := strings.Split(s, ",") - for _, labelChunk := range labelChunks { - split := strings.Split(labelChunk, ":") - m = append(m, Label{Name: split[0], Value: split[1]}) - } - // Order of the k/v labels matters, don't assume we'll always receive them already sorted. - sort.Sort(m) - h := m.Hash() - if _, ok := hashes[h]; ok { - continue - } - mets = append(mets, m) - hashes[h] = struct{}{} - i++ - } - return mets, nil -} - func BenchmarkLabelSetFromMap(b *testing.B) { m := map[string]string{ "job": "node", diff --git a/querier_test.go b/querier_test.go index 2eb34f450..88b8f088e 100644 --- a/querier_test.go +++ b/querier_test.go @@ -1248,7 +1248,7 @@ func BenchmarkMergedSeriesSet(b *testing.B) { } { for _, j := range []int{1, 2, 4, 8, 16, 32} { b.Run(fmt.Sprintf("series=%d,blocks=%d", k, j), func(b *testing.B) { - lbls, err := readPrometheusLabels("testdata/1m.series", k) + lbls, err := labels.ReadLabels("testdata/1m.series", k) testutil.Ok(b, err) sort.Sort(labels.Slice(lbls)) @@ -1299,7 +1299,7 @@ func (cr mockChunkReader) Close() error { } func TestDeletedIterator(t *testing.T) { - chk := chunks.NewXORChunk() + chk := chunkenc.NewXORChunk() app, err := chk.Appender() testutil.Ok(t, err) // Insert random stuff from (0, 1000). diff --git a/wal_test.go b/wal_test.go index 664c2010f..594fb54d4 100644 --- a/wal_test.go +++ b/wal_test.go @@ -22,6 +22,7 @@ import ( "github.com/go-kit/kit/log" "github.com/prometheus/tsdb/fileutil" + "github.com/prometheus/tsdb/labels" "github.com/prometheus/tsdb/testutil" ) @@ -71,7 +72,7 @@ func TestSegmentWAL_Truncate(t *testing.T) { numMetrics = 20000 batch = 100 ) - series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics) + series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics) testutil.Ok(t, err) dir, err := ioutil.TempDir("", "test_wal_log_truncate") @@ -150,7 +151,7 @@ func TestSegmentWAL_Log_Restore(t *testing.T) { ) // Generate testing data. It does not make semantical sense but // for the purpose of this test. - series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics) + series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics) testutil.Ok(t, err) dir, err := ioutil.TempDir("", "test_wal_log_restore")