github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

move label read function int labels pkg

pull/5805/head
Fabian Reinartz 7 years ago
parent
67f0ca8f0e
commit
adb6be73dd
7 changed files with 71 additions and 145 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      block_test.go
  2. 55
      head_test.go
  3. 57
      index/index_test.go
  4. 49
      labels/labels.go
  5. 42
      labels/labels_test.go
  6. 4
      querier_test.go
  7. 5
      wal_test.go

4
block_test.go
Unescape View File

@ -45,8 +45,8 @@ func createEmptyBlock(t *testing.T, dir string) *Block {
testutil.Ok(t, writeMetaFile(dir, &BlockMeta{}))
ir, err := index.NewWriter(filepath.Join(dir, indexFilename))
Ok(t, err)
Ok(t, ir.Close())
testutil.Ok(t, err)
testutil.Ok(t, ir.Close())
testutil.Ok(t, os.MkdirAll(chunkDir(dir), 0777))

55
head_test.go
Unescape View File

@ -14,22 +14,18 @@
package tsdb
import (
"bufio"
"math/rand"
"os"
"sort"
"strings"
"testing"
"github.com/pkg/errors"
"github.com/prometheus/tsdb/chunkenc"
"github.com/prometheus/tsdb/chunks"
"github.com/prometheus/tsdb/index"
"github.com/prometheus/tsdb/labels"
"github.com/prometheus/tsdb/testutil"
)
func BenchmarkCreateSeries(b *testing.B) {
lbls, err := readPrometheusLabels("testdata/all.series", b.N)
lbls, err := labels.ReadLabels("testdata/all.series", b.N)
testutil.Ok(b, err)
h, err := NewHead(nil, nil, nil, 10000)
@ -46,49 +42,6 @@ func BenchmarkCreateSeries(b *testing.B) {
}
}
func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
var mets []labels.Labels
hashes := map[uint64]struct{}{}
i := 0
for scanner.Scan() && i < n {
m := make(labels.Labels, 0, 10)
r := strings.NewReplacer("\"", "", "{", "", "}", "")
s := r.Replace(scanner.Text())
labelChunks := strings.Split(s, ",")
for _, labelChunk := range labelChunks {
split := strings.Split(labelChunk, ":")
m = append(m, labels.Label{Name: split[0], Value: split[1]})
}
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort.Sort(m)
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
if err != nil {
return nil, err
}
if i != n {
return mets, errors.Errorf("requested %d metrics but found %d", n, i)
}
return mets, nil
}
type memoryWAL struct {
nopWAL
entries []interface{}
@ -727,7 +680,7 @@ func TestGCChunkAccess(t *testing.T) {
idx := h.indexRange(0, 1500)
var (
lset labels.Labels
chunks []ChunkMeta
chunks []chunks.Meta
)
testutil.Ok(t, idx.Series(1, &lset, &chunks))
@ -767,7 +720,7 @@ func TestGCSeriesAccess(t *testing.T) {
idx := h.indexRange(0, 2000)
var (
lset labels.Labels
chunks []ChunkMeta
chunks []chunks.Meta
)
testutil.Ok(t, idx.Series(1, &lset, &chunks))

57
index/index_test.go
Unescape View File

@ -20,11 +20,8 @@ import (
"path/filepath"
"sort"
"testing"
"unsafe"
"github.com/pkg/errors"
promlabels "github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/pkg/textparse"
"github.com/prometheus/tsdb/chunkenc"
"github.com/prometheus/tsdb/chunks"
"github.com/prometheus/tsdb/labels"
@ -242,7 +239,7 @@ func TestPersistence_index_e2e(t *testing.T) {
testutil.Ok(t, err)
defer os.RemoveAll(dir)
lbls, err := readPrometheusLabels("testdata/20kseries.json", 20000)
lbls, err := labels.ReadLabels("../testdata/20kseries.json", 20000)
testutil.Ok(t, err)
// Sort labels as the index writer expects series in sorted order.
@ -307,7 +304,11 @@ func TestPersistence_index_e2e(t *testing.T) {
}
for k, v := range values {
vals := v.slice()
var vals []string
for e := range v {
vals = append(vals, e)
}
sort.Strings(vals)
testutil.Ok(t, iw.WriteLabelIndex([]string{k}, vals))
testutil.Ok(t, mi.WriteLabelIndex([]string{k}, vals))
@ -322,15 +323,15 @@ func TestPersistence_index_e2e(t *testing.T) {
mi.WritePostings("", "", newListPostings(all))
for l := range postings.m {
err = iw.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value))
err = iw.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value))
testutil.Ok(t, err)
mi.WritePostings(l.Name, l.Value, postings.get(l.Name, l.Value))
mi.WritePostings(l.Name, l.Value, postings.Get(l.Name, l.Value))
}
err = iw.Close()
testutil.Ok(t, err)
ir, err := NewFileIndexReader(filepath.Join(dir, "index"))
ir, err := NewFileReader(filepath.Join(dir, "index"))
testutil.Ok(t, err)
for p := range mi.postings {
@ -359,7 +360,7 @@ func TestPersistence_index_e2e(t *testing.T) {
}
for k, v := range mi.labelIndex {
tplsExp, err := newStringTuples(v, 1)
tplsExp, err := NewStringTuples(v, 1)
testutil.Ok(t, err)
tplsRes, err := ir.LabelValues(k)
@ -379,41 +380,3 @@ func TestPersistence_index_e2e(t *testing.T) {
testutil.Ok(t, ir.Close())
}
func readPrometheusLabels(fn string, n int) ([]labels.Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
p := textparse.New(b)
i := 0
var mets []labels.Labels
hashes := map[uint64]struct{}{}
for p.Next() && i < n {
m := make(labels.Labels, 0, 10)
p.Metric((*promlabels.Labels)(unsafe.Pointer(&m)))
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
if err := p.Err(); err != nil {
return nil, err
}
if i != n {
return mets, errors.Errorf("requested %d metrics but found %d", n, i)
}
return mets, nil
}

49
labels/labels.go
Unescape View File

@ -14,12 +14,15 @@
package labels
import (
"bufio"
"bytes"
"os"
"sort"
"strconv"
"strings"
"github.com/cespare/xxhash"
"github.com/pkg/errors"
)
const sep = '\xff'
@ -161,3 +164,49 @@ type Slice []Labels
func (s Slice) Len() int { return len(s) }
func (s Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s Slice) Less(i, j int) bool { return Compare(s[i], s[j]) < 0 }
// ReadLabels reads up to n label sets in a JSON formatted file fn. It is mostly useful
// to load testing data.
func ReadLabels(fn string, n int) ([]Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
var mets []Labels
hashes := map[uint64]struct{}{}
i := 0
for scanner.Scan() && i < n {
m := make(Labels, 0, 10)
r := strings.NewReplacer("\"", "", "{", "", "}", "")
s := r.Replace(scanner.Text())
labelChunks := strings.Split(s, ",")
for _, labelChunk := range labelChunks {
split := strings.Split(labelChunk, ":")
m = append(m, Label{Name: split[0], Value: split[1]})
}
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort.Sort(m)
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
if err != nil {
return nil, err
}
if i != n {
return mets, errors.Errorf("requested %d metrics but found %d", n, i)
}
return mets, nil
}

42
labels/labels_test.go
Unescape View File

@ -14,12 +14,9 @@
package labels
import (
"bufio"
"fmt"
"math/rand"
"os"
"sort"
"strings"
"testing"
"github.com/prometheus/tsdb/testutil"
@ -90,7 +87,7 @@ func TestCompareAndEquals(t *testing.T) {
}
func BenchmarkSliceSort(b *testing.B) {
lbls, err := readPrometheusLabels("../testdata/1m.series", 900000)
lbls, err := ReadLabels("../testdata/1m.series", 900000)
testutil.Ok(b, err)
for len(lbls) < 20e6 {
@ -119,43 +116,6 @@ func BenchmarkSliceSort(b *testing.B) {
}
}
func readPrometheusLabels(fn string, n int) ([]Labels, error) {
f, err := os.Open(fn)
if err != nil {
return nil, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
var mets []Labels
hashes := map[uint64]struct{}{}
i := 0
for scanner.Scan() && i < n {
m := make(Labels, 0, 10)
r := strings.NewReplacer("\"", "", "{", "", "}", "")
s := r.Replace(scanner.Text())
labelChunks := strings.Split(s, ",")
for _, labelChunk := range labelChunks {
split := strings.Split(labelChunk, ":")
m = append(m, Label{Name: split[0], Value: split[1]})
}
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
sort.Sort(m)
h := m.Hash()
if _, ok := hashes[h]; ok {
continue
}
mets = append(mets, m)
hashes[h] = struct{}{}
i++
}
return mets, nil
}
func BenchmarkLabelSetFromMap(b *testing.B) {
m := map[string]string{
"job": "node",

4
querier_test.go
Unescape View File

@ -1248,7 +1248,7 @@ func BenchmarkMergedSeriesSet(b *testing.B) {
} {
for _, j := range []int{1, 2, 4, 8, 16, 32} {
b.Run(fmt.Sprintf("series=%d,blocks=%d", k, j), func(b *testing.B) {
lbls, err := readPrometheusLabels("testdata/1m.series", k)
lbls, err := labels.ReadLabels("testdata/1m.series", k)
testutil.Ok(b, err)
sort.Sort(labels.Slice(lbls))
@ -1299,7 +1299,7 @@ func (cr mockChunkReader) Close() error {
}
func TestDeletedIterator(t *testing.T) {
chk := chunks.NewXORChunk()
chk := chunkenc.NewXORChunk()
app, err := chk.Appender()
testutil.Ok(t, err)
// Insert random stuff from (0, 1000).

5
wal_test.go
Unescape View File

@ -22,6 +22,7 @@ import (
"github.com/go-kit/kit/log"
"github.com/prometheus/tsdb/fileutil"
"github.com/prometheus/tsdb/labels"
"github.com/prometheus/tsdb/testutil"
)
@ -71,7 +72,7 @@ func TestSegmentWAL_Truncate(t *testing.T) {
numMetrics = 20000
batch = 100
)
series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics)
series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics)
testutil.Ok(t, err)
dir, err := ioutil.TempDir("", "test_wal_log_truncate")
@ -150,7 +151,7 @@ func TestSegmentWAL_Log_Restore(t *testing.T) {
)
// Generate testing data. It does not make semantical sense but
// for the purpose of this test.
series, err := readPrometheusLabels("testdata/20kseries.json", numMetrics)
series, err := labels.ReadLabels("testdata/20kseries.json", numMetrics)
testutil.Ok(t, err)
dir, err := ioutil.TempDir("", "test_wal_log_restore")

Write Preview
Loading…
Cancel
Save