pkg/textparse: add tests and method to retrieve full labels

pkg/textparse/lex.l

@@ -27,6 +27,8 @@ func (l *lexer) Lex() int {
         return eof
     }
     c := l.b[l.i]
+
+    l.offsets = l.offsets[:0]
 %}
 
 D     [0-9]
@@ -47,19 +49,25 @@ M     [a-zA-Z_:]
 #[^\r\n]*\n          l.mstart = l.i
 [\r\n \t]+           l.mstart = l.i
 
-{L}({L}|{D})*\{      s = lstateLName
+{L}({L}|{D})*\{      s = lstateLabels
+                     l.offsets = append(l.offsets, l.i-1)
 {L}({L}|{D})*        s = lstateValue
                      l.mend = l.i
+                     l.offsets = append(l.offsets, l.i)
 
 <lstateLabels>[ \t]+
 <lstateLabels>\}       s = lstateValue
                        l.mend = l.i
 <lstateLabels>,?       s = lstateLName
+                       l.offsets = append(l.offsets, l.i)
 
 <lstateLName>{M}({M}|{D})*=    s = lstateLValue
+                               l.offsets = append(l.offsets, l.i-1)
 
 <lstateLValue>\"(\\.|[^\\"])*\"       s = lstateLabels
+                                      l.offsets = append(l.offsets, l.i-1)
 <lstateLValue>\'(\\.|[^\\'])*\'       s = lstateLabels
+                                      l.offsets = append(l.offsets, l.i-1)
 
 <lstateValue>[ \t]+             l.vstart = l.i
 <lstateValue>(NaN)              l.val = math.NaN()

pkg/textparse/lex.l.go

@@ -28,6 +28,8 @@ func (l *lexer) Lex() int {
 	}
 	c := l.b[l.i]
 
+	l.offsets = l.offsets[:0]
+
 yystate0:
 
 	switch yyt := s; yyt {
@@ -309,13 +311,15 @@ yyrule3: // [\r\n \t]+
 	}
 yyrule4: // {L}({L}|{D})*\{
 	{
-		s = lstateLName
+		s = lstateLabels
+		l.offsets = append(l.offsets, l.i-1)
 		goto yystate0
 	}
 yyrule5: // {L}({L}|{D})*
 	{
 		s = lstateValue
 		l.mend = l.i
+		l.offsets = append(l.offsets, l.i)
 		goto yystate0
 	}
 yyrule6: // [ \t]+
@@ -330,21 +334,25 @@ yyrule7: // \}
 yyrule8: // ,?
 	{
 		s = lstateLName
+		l.offsets = append(l.offsets, l.i)
 		goto yystate0
 	}
 yyrule9: // {M}({M}|{D})*=
 	{
 		s = lstateLValue
+		l.offsets = append(l.offsets, l.i-1)
 		goto yystate0
 	}
 yyrule10: // \"(\\.|[^\\"])*\"
 	{
 		s = lstateLabels
+		l.offsets = append(l.offsets, l.i-1)
 		goto yystate0
 	}
 yyrule11: // \'(\\.|[^\\'])*\'
 	{
 		s = lstateLabels
+		l.offsets = append(l.offsets, l.i-1)
 		goto yystate0
 	}
 yyrule12: // [ \t]+

pkg/textparse/parse.go

@@ -5,9 +5,10 @@ import (
 	"errors"
 	"io"
 	"reflect"
+	"sort"
 	"unsafe"
 
-	"k8s.io/client-go/pkg/labels"
+	"github.com/prometheus/prometheus/pkg/labels"
 )
 
 type lexer struct {
@@ -15,9 +16,10 @@ type lexer struct {
 	i      int
 	vstart int
 
-	mstart, mend int
 	err          error
 	val          float64
+	offsets      []int
+	mstart, mend int
 }
 
 const eof = 0
@@ -70,8 +72,22 @@ func (p *Parser) Err() error {
 	return p.l.err
 }
 
-func (p *Parser) Metric() labels.Labels {
+func (p *Parser) Metric(l *labels.Labels) {
-	return nil
+	*l = append(*l, labels.Label{
+		Name:  labels.MetricName,
+		Value: string(p.l.b[p.l.mstart:p.l.offsets[0]]),
+	})
+
+	for i := 1; i < len(p.l.offsets); i += 3 {
+		a, b, c := p.l.offsets[i], p.l.offsets[i+1], p.l.offsets[i+2]
+
+		*l = append(*l, labels.Label{
+			Name:  string(p.l.b[a:b]),
+			Value: string(p.l.b[b+2 : c]),
+		})
+	}
+
+	sort.Sort((*l)[1:])
 }
 
 func yoloString(b []byte) string {

pkg/textparse/parse_test.go

@@ -10,9 +10,69 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestParse(t *testing.T) {
+	input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"} 4.9351e-05
+go_gc_duration_seconds{quantile="0.25"} 7.424100000000001e-05
+go_gc_duration_seconds{quantile="0.5",a="b"} 8.3835e-05
+go_gc_duration_seconds_count 99
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines 33`
+
+	exp := []struct {
+		lset labels.Labels
+		m    string
+		v    float64
+	}{
+		{
+			m:    `go_gc_duration_seconds{quantile="0"}`,
+			v:    4.9351e-05,
+			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0"),
+		}, {
+			m:    `go_gc_duration_seconds{quantile="0.25"}`,
+			v:    7.424100000000001e-05,
+			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.25"),
+		}, {
+			m:    `go_gc_duration_seconds{quantile="0.5",a="b"}`,
+			v:    8.3835e-05,
+			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.5", "a", "b"),
+		}, {
+			m:    `go_gc_duration_seconds_count`,
+			v:    99,
+			lset: labels.FromStrings("__name__", "go_gc_duration_seconds_count"),
+		}, {
+			m:    `go_goroutines`,
+			v:    33,
+			lset: labels.FromStrings("__name__", "go_goroutines"),
+		},
+	}
+
+	p := New([]byte(input))
+	i := 0
+
+	var res labels.Labels
+
+	for p.Next() {
+		m, _, v := p.At()
+
+		p.Metric(&res)
+
+		require.Equal(t, exp[i].m, string(m))
+		require.Equal(t, exp[i].v, v)
+		require.Equal(t, exp[i].lset, res)
+
+		i++
+		res = res[:0]
+	}
+
+	require.NoError(t, p.Err())
+
+}
+
 func BenchmarkParse(b *testing.B) {
 	f, err := os.Open("testdata.txt")
-	// f, err := os.Open("../../../../fabxc/tsdb/cmd/tsdb/testdata.1m")
 	require.NoError(b, err)
 	defer f.Close()
 
@@ -21,7 +81,7 @@ func BenchmarkParse(b *testing.B) {
 	buf, err := ioutil.ReadAll(f)
 	require.NoError(b, err)
 
-	for i := 0; i*1e6 < b.N; i++ {
+	for i := 0; i*527 < b.N; i++ {
 		tbuf = append(tbuf, buf...)
 		tbuf = append(tbuf, '\n')
 	}
@@ -30,7 +90,6 @@ func BenchmarkParse(b *testing.B) {
 	i := 0
 	var m labels.Labels
 	total := 0
-	// res := make(labels.Labels, 0, 5)
 
 	b.ResetTimer()
 	b.ReportAllocs()
@@ -39,13 +98,12 @@ func BenchmarkParse(b *testing.B) {
 	for p.Next() && i < b.N {
 		p.At()
 
-		// res = res[:0]
+		res := make(labels.Labels, 0, 5)
+		p.Metric(&res)
 
-		// err = ParseMetric(mb, &res)
+		total += len(res)
-		// require.NoError(b, err)
-
-		// total += len(res)
 		i++
+		res = res[:0]
 	}
 	_ = m
 	fmt.Println(total)