Merge pull request #2661 from prometheus/texts

pkg/textparse: implement timestamp parsing

Makefile

@@ -39,11 +39,11 @@ check_license:
 # TODO(fabxc): example tests temporarily removed.
 test-short:
 	@echo ">> running short tests"
-	@$(GO) test $(shell $(GO) list ./... | grep -v /vendor/ | grep -v examples)
+	@$(GO) test -short $(shell $(GO) list ./... | grep -v /vendor/ | grep -v examples)
 
 test:
 	@echo ">> running all tests"
-	@$(GO) test $(pkgs)
+	@$(GO) test $(shell $(GO) list ./... | grep -v /vendor/ | grep -v examples)
 
 format:
 	@echo ">> formatting code"

pkg/textparse/lex.l

@@ -28,6 +28,7 @@ func (l *lexer) Lex() int {
     const (
         lstateInit = iota
         lstateValue
+        lstateTimestamp
         lstateLabels
         lstateLName
         lstateLValue
@@ -39,6 +40,8 @@ func (l *lexer) Lex() int {
     }
     c := l.b[l.i]
 
+    l.ts = nil
+    l.mstart = l.nextMstart
     l.offsets = l.offsets[:0]
 %}
 
@@ -46,7 +49,7 @@ D     [0-9]
 L     [a-zA-Z_]
 M     [a-zA-Z_:]
 
-%x lstateValue lstateLabels lstateLName lstateLValue
+%x lstateValue lstateTimestamp lstateLabels lstateLName lstateLValue
 
 
 %yyc c
@@ -82,15 +85,27 @@ M     [a-zA-Z_:]
 
 <lstateValue>[ \t]+                   l.vstart = l.i
 <lstateValue>(NaN)                    l.val = math.NaN()
-                                      return 1
+                                      s = lstateTimestamp
 <lstateValue>[^\n \t\r]+              // We don't parse strictly correct floats as the conversion
                                       // repeats the effort anyway.
                                       l.val, l.err = strconv.ParseFloat(yoloString(l.b[l.vstart:l.i]), 64)
                                       if l.err != nil {
                                           return -1
                                       }
+                                      s = lstateTimestamp
+
+<lstateTimestamp>[ \t]+               l.tstart = l.i         
+<lstateTimestamp>{D}+                 ts, err := strconv.ParseInt(yoloString(l.b[l.tstart:l.i]), 10, 64)
+                                      if err != nil {
+                                          l.err = err
+                                          return -1
+                                      }
+                                      l.ts = &ts
+<lstateTimestamp>[\r\n]+              l.nextMstart = l.i
                                       return 1
+<lstateTimestamp>\0                   return 1              
    
 %%
+    l.err = fmt.Errorf("no token found")
     return -1
 }

pkg/textparse/lex.l.go

@@ -28,6 +28,7 @@ func (l *lexer) Lex() int {
 	const (
 		lstateInit = iota
 		lstateValue
+		lstateTimestamp
 		lstateLabels
 		lstateLName
 		lstateLValue
@@ -39,6 +40,8 @@ func (l *lexer) Lex() int {
 	}
 	c := l.b[l.i]
 
+	l.ts = nil
+	l.mstart = l.nextMstart
 	l.offsets = l.offsets[:0]
 
 yystate0:
@@ -50,12 +53,14 @@ yystate0:
 		goto yystart1
 	case 1: // start condition: lstateValue
 		goto yystart8
-	case 2: // start condition: lstateLabels
+	case 2: // start condition: lstateTimestamp
 		goto yystart14
-	case 3: // start condition: lstateLName
+	case 3: // start condition: lstateLabels
-		goto yystart18
+		goto yystart19
-	case 4: // start condition: lstateLValue
+	case 4: // start condition: lstateLName
-		goto yystart21
+		goto yystart23
+	case 5: // start condition: lstateLValue
+		goto yystart26
 	}
 
 	goto yystate0 // silence unused label error
@@ -189,121 +194,169 @@ yystate14:
 yystart14:
 	switch {
 	default:
-		goto yyrule8
+		goto yyabort
-	case c == ',':
+	case c == '\n' || c == '\r':
-		goto yystate16
-	case c == '\t' || c == ' ':
-		goto yystate15
-	case c == '}':
 		goto yystate17
+	case c == '\t' || c == ' ':
+		goto yystate16
+	case c == '\x00':
+		goto yystate15
+	case c >= '0' && c <= '9':
+		goto yystate18
 	}
 
 yystate15:
+	c = l.next()
+	goto yyrule18
+
+yystate16:
+	c = l.next()
+	switch {
+	default:
+		goto yyrule15
+	case c == '\t' || c == ' ':
+		goto yystate16
+	}
+
+yystate17:
+	c = l.next()
+	switch {
+	default:
+		goto yyrule17
+	case c == '\n' || c == '\r':
+		goto yystate17
+	}
+
+yystate18:
+	c = l.next()
+	switch {
+	default:
+		goto yyrule16
+	case c >= '0' && c <= '9':
+		goto yystate18
+	}
+
+	goto yystate19 // silence unused label error
+yystate19:
+	c = l.next()
+yystart19:
+	switch {
+	default:
+		goto yyrule8
+	case c == ',':
+		goto yystate21
+	case c == '\t' || c == ' ':
+		goto yystate20
+	case c == '}':
+		goto yystate22
+	}
+
+yystate20:
 	c = l.next()
 	switch {
 	default:
 		goto yyrule6
 	case c == '\t' || c == ' ':
-		goto yystate15
+		goto yystate20
 	}
 
-yystate16:
+yystate21:
 	c = l.next()
 	goto yyrule8
 
-yystate17:
+yystate22:
 	c = l.next()
 	goto yyrule7
 
-	goto yystate18 // silence unused label error
+	goto yystate23 // silence unused label error
-yystate18:
+yystate23:
 	c = l.next()
-yystart18:
+yystart23:
 	switch {
 	default:
 		goto yyabort
 	case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate19
-	}
-
-yystate19:
-	c = l.next()
-	switch {
-	default:
-		goto yyabort
-	case c == '=':
-		goto yystate20
-	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
-		goto yystate19
-	}
-
-yystate20:
-	c = l.next()
-	goto yyrule9
-
-	goto yystate21 // silence unused label error
-yystate21:
-	c = l.next()
-yystart21:
-	switch {
-	default:
-		goto yyabort
-	case c == '"':
-		goto yystate22
-	case c == '\'':
-		goto yystate25
-	}
-
-yystate22:
-	c = l.next()
-	switch {
-	default:
-		goto yyabort
-	case c == '"':
-		goto yystate23
-	case c == '\\':
 		goto yystate24
-	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
-		goto yystate22
 	}
 
-yystate23:
-	c = l.next()
-	goto yyrule10
-
 yystate24:
 	c = l.next()
 	switch {
 	default:
 		goto yyabort
-	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
+	case c == '=':
-		goto yystate22
+		goto yystate25
+	case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z':
+		goto yystate24
 	}
 
 yystate25:
 	c = l.next()
+	goto yyrule9
+
+	goto yystate26 // silence unused label error
+yystate26:
+	c = l.next()
+yystart26:
 	switch {
 	default:
 		goto yyabort
-	case c == '\'':
+	case c == '"':
-		goto yystate26
-	case c == '\\':
 		goto yystate27
-	case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ':
+	case c == '\'':
-		goto yystate25
+		goto yystate30
 	}
 
-yystate26:
-	c = l.next()
-	goto yyrule11
-
 yystate27:
+	c = l.next()
+	switch {
+	default:
+		goto yyabort
+	case c == '"':
+		goto yystate28
+	case c == '\\':
+		goto yystate29
+	case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate27
+	}
+
+yystate28:
+	c = l.next()
+	goto yyrule10
+
+yystate29:
 	c = l.next()
 	switch {
 	default:
 		goto yyabort
 	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
-		goto yystate25
+		goto yystate27
+	}
+
+yystate30:
+	c = l.next()
+	switch {
+	default:
+		goto yyabort
+	case c == '\'':
+		goto yystate31
+	case c == '\\':
+		goto yystate32
+	case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ':
+		goto yystate30
+	}
+
+yystate31:
+	c = l.next()
+	goto yyrule11
+
+yystate32:
+	c = l.next()
+	switch {
+	default:
+		goto yyabort
+	case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ':
+		goto yystate30
 	}
 
 yyrule1: // \0
@@ -374,7 +427,8 @@ yyrule12: // [ \t]+
 yyrule13: // (NaN)
 	{
 		l.val = math.NaN()
-		return 1
+		s = lstateTimestamp
+		goto yystate0
 	}
 yyrule14: // [^\n \t\r]+
 	{
@@ -384,6 +438,31 @@ yyrule14: // [^\n \t\r]+
 		if l.err != nil {
 			return -1
 		}
+		s = lstateTimestamp
+		goto yystate0
+	}
+yyrule15: // [ \t]+
+	{
+		l.tstart = l.i
+		goto yystate0
+	}
+yyrule16: // {D}+
+	{
+		ts, err := strconv.ParseInt(yoloString(l.b[l.tstart:l.i]), 10, 64)
+		if err != nil {
+			l.err = err
+			return -1
+		}
+		l.ts = &ts
+		goto yystate0
+	}
+yyrule17: // [\r\n]+
+	{
+		l.nextMstart = l.i
+		return 1
+	}
+yyrule18: // \0
+	{
 		return 1
 
 	}
@@ -392,5 +471,6 @@ yyrule14: // [^\n \t\r]+
 	goto yyabort // silence unused label error
 
 yyabort: // no lexem recognized
+	l.err = fmt.Errorf("no token found")
 	return -1
 }

pkg/textparse/parse.go

@@ -30,11 +30,14 @@ type lexer struct {
 	b      []byte
 	i      int
 	vstart int
+	tstart int
 
 	err          error
 	val          float64
+	ts           *int64
 	offsets      []int
 	mstart, mend int
+	nextMstart   int
 }
 
 const eof = 0
@@ -70,7 +73,7 @@ func New(b []byte) *Parser {
 // more samples were read or an error occurred.
 func (p *Parser) Next() bool {
 	switch p.l.Lex() {
-	case 0, -1:
+	case -1, eof:
 		return false
 	case 1:
 		return true
@@ -81,7 +84,7 @@ func (p *Parser) Next() bool {
 // At returns the bytes of the metric, the timestamp if set, and the value
 // of the current sample.
 func (p *Parser) At() ([]byte, *int64, float64) {
-	return p.l.b[p.l.mstart:p.l.mend], nil, p.l.val
+	return p.l.b[p.l.mstart:p.l.mend], p.l.ts, p.l.val
 }
 
 // Err returns the current error.

pkg/textparse/parse_test.go

@@ -36,11 +36,14 @@ go_gc_duration_seconds{quantile="0.5",a="b"} 8.3835e-05
 go_gc_duration_seconds_count 99
 # HELP go_goroutines Number of goroutines that currently exist.
 # TYPE go_goroutines gauge
-go_goroutines 33`
+go_goroutines 33  	123123`
+
+	int64p := func(x int64) *int64 { return &x }
 
 	exp := []struct {
 		lset labels.Labels
 		m    string
+		t    *int64
 		v    float64
 	}{
 		{
@@ -62,6 +65,7 @@ go_goroutines 33`
 		}, {
 			m:    `go_goroutines`,
 			v:    33,
+			t:    int64p(123123),
 			lset: labels.FromStrings("__name__", "go_goroutines"),
 		},
 	}
@@ -72,11 +76,12 @@ go_goroutines 33`
 	var res labels.Labels
 
 	for p.Next() {
-		m, _, v := p.At()
+		m, ts, v := p.At()
 
 		p.Metric(&res)
 
 		require.Equal(t, exp[i].m, string(m))
+		require.Equal(t, exp[i].t, ts)
 		require.Equal(t, exp[i].v, v)
 		require.Equal(t, exp[i].lset, res)
 
@@ -85,6 +90,7 @@ go_goroutines 33`
 	}
 
 	require.NoError(t, p.Err())
+	require.Equal(t, len(exp), i)
 
 }