diff --git a/Makefile b/Makefile index 1247bd1f0..a07f67efe 100644 --- a/Makefile +++ b/Makefile @@ -39,11 +39,11 @@ check_license: # TODO(fabxc): example tests temporarily removed. test-short: @echo ">> running short tests" - @$(GO) test $(shell $(GO) list ./... | grep -v /vendor/ | grep -v examples) + @$(GO) test -short $(shell $(GO) list ./... | grep -v /vendor/ | grep -v examples) test: @echo ">> running all tests" - @$(GO) test $(pkgs) + @$(GO) test $(shell $(GO) list ./... | grep -v /vendor/ | grep -v examples) format: @echo ">> formatting code" diff --git a/pkg/textparse/lex.l b/pkg/textparse/lex.l index f875355b7..3decbb1ca 100644 --- a/pkg/textparse/lex.l +++ b/pkg/textparse/lex.l @@ -28,6 +28,7 @@ func (l *lexer) Lex() int { const ( lstateInit = iota lstateValue + lstateTimestamp lstateLabels lstateLName lstateLValue @@ -39,6 +40,8 @@ func (l *lexer) Lex() int { } c := l.b[l.i] + l.ts = nil + l.mstart = l.nextMstart l.offsets = l.offsets[:0] %} @@ -46,7 +49,7 @@ D [0-9] L [a-zA-Z_] M [a-zA-Z_:] -%x lstateValue lstateLabels lstateLName lstateLValue +%x lstateValue lstateTimestamp lstateLabels lstateLName lstateLValue %yyc c @@ -82,15 +85,27 @@ M [a-zA-Z_:] [ \t]+ l.vstart = l.i (NaN) l.val = math.NaN() - return 1 + s = lstateTimestamp [^\n \t\r]+ // We don't parse strictly correct floats as the conversion // repeats the effort anyway. l.val, l.err = strconv.ParseFloat(yoloString(l.b[l.vstart:l.i]), 64) if l.err != nil { return -1 } + s = lstateTimestamp + +[ \t]+ l.tstart = l.i +{D}+ ts, err := strconv.ParseInt(yoloString(l.b[l.tstart:l.i]), 10, 64) + if err != nil { + l.err = err + return -1 + } + l.ts = &ts +[\r\n]+ l.nextMstart = l.i return 1 +\0 return 1 %% + l.err = fmt.Errorf("no token found") return -1 } diff --git a/pkg/textparse/lex.l.go b/pkg/textparse/lex.l.go index c1b3e7d25..11ebde484 100644 --- a/pkg/textparse/lex.l.go +++ b/pkg/textparse/lex.l.go @@ -28,6 +28,7 @@ func (l *lexer) Lex() int { const ( lstateInit = iota lstateValue + lstateTimestamp lstateLabels lstateLName lstateLValue @@ -39,6 +40,8 @@ func (l *lexer) Lex() int { } c := l.b[l.i] + l.ts = nil + l.mstart = l.nextMstart l.offsets = l.offsets[:0] yystate0: @@ -50,12 +53,14 @@ yystate0: goto yystart1 case 1: // start condition: lstateValue goto yystart8 - case 2: // start condition: lstateLabels + case 2: // start condition: lstateTimestamp goto yystart14 - case 3: // start condition: lstateLName - goto yystart18 - case 4: // start condition: lstateLValue - goto yystart21 + case 3: // start condition: lstateLabels + goto yystart19 + case 4: // start condition: lstateLName + goto yystart23 + case 5: // start condition: lstateLValue + goto yystart26 } goto yystate0 // silence unused label error @@ -189,121 +194,169 @@ yystate14: yystart14: switch { default: - goto yyrule8 - case c == ',': - goto yystate16 - case c == '\t' || c == ' ': - goto yystate15 - case c == '}': + goto yyabort + case c == '\n' || c == '\r': goto yystate17 + case c == '\t' || c == ' ': + goto yystate16 + case c == '\x00': + goto yystate15 + case c >= '0' && c <= '9': + goto yystate18 } yystate15: + c = l.next() + goto yyrule18 + +yystate16: + c = l.next() + switch { + default: + goto yyrule15 + case c == '\t' || c == ' ': + goto yystate16 + } + +yystate17: + c = l.next() + switch { + default: + goto yyrule17 + case c == '\n' || c == '\r': + goto yystate17 + } + +yystate18: + c = l.next() + switch { + default: + goto yyrule16 + case c >= '0' && c <= '9': + goto yystate18 + } + + goto yystate19 // silence unused label error +yystate19: + c = l.next() +yystart19: + switch { + default: + goto yyrule8 + case c == ',': + goto yystate21 + case c == '\t' || c == ' ': + goto yystate20 + case c == '}': + goto yystate22 + } + +yystate20: c = l.next() switch { default: goto yyrule6 case c == '\t' || c == ' ': - goto yystate15 + goto yystate20 } -yystate16: +yystate21: c = l.next() goto yyrule8 -yystate17: +yystate22: c = l.next() goto yyrule7 - goto yystate18 // silence unused label error -yystate18: + goto yystate23 // silence unused label error +yystate23: c = l.next() -yystart18: +yystart23: switch { default: goto yyabort case c == ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': - goto yystate19 - } - -yystate19: - c = l.next() - switch { - default: - goto yyabort - case c == '=': - goto yystate20 - case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': - goto yystate19 - } - -yystate20: - c = l.next() - goto yyrule9 - - goto yystate21 // silence unused label error -yystate21: - c = l.next() -yystart21: - switch { - default: - goto yyabort - case c == '"': - goto yystate22 - case c == '\'': - goto yystate25 - } - -yystate22: - c = l.next() - switch { - default: - goto yyabort - case c == '"': - goto yystate23 - case c == '\\': goto yystate24 - case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': - goto yystate22 } -yystate23: - c = l.next() - goto yyrule10 - yystate24: c = l.next() switch { default: goto yyabort - case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': - goto yystate22 + case c == '=': + goto yystate25 + case c >= '0' && c <= ':' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yystate24 } yystate25: c = l.next() + goto yyrule9 + + goto yystate26 // silence unused label error +yystate26: + c = l.next() +yystart26: switch { default: goto yyabort - case c == '\'': - goto yystate26 - case c == '\\': + case c == '"': goto yystate27 - case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ': - goto yystate25 + case c == '\'': + goto yystate30 } -yystate26: - c = l.next() - goto yyrule11 - yystate27: + c = l.next() + switch { + default: + goto yyabort + case c == '"': + goto yystate28 + case c == '\\': + goto yystate29 + case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate27 + } + +yystate28: + c = l.next() + goto yyrule10 + +yystate29: c = l.next() switch { default: goto yyabort case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': - goto yystate25 + goto yystate27 + } + +yystate30: + c = l.next() + switch { + default: + goto yyabort + case c == '\'': + goto yystate31 + case c == '\\': + goto yystate32 + case c >= '\x01' && c <= '&' || c >= '(' && c <= '[' || c >= ']' && c <= 'ÿ': + goto yystate30 + } + +yystate31: + c = l.next() + goto yyrule11 + +yystate32: + c = l.next() + switch { + default: + goto yyabort + case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': + goto yystate30 } yyrule1: // \0 @@ -374,7 +427,8 @@ yyrule12: // [ \t]+ yyrule13: // (NaN) { l.val = math.NaN() - return 1 + s = lstateTimestamp + goto yystate0 } yyrule14: // [^\n \t\r]+ { @@ -384,6 +438,31 @@ yyrule14: // [^\n \t\r]+ if l.err != nil { return -1 } + s = lstateTimestamp + goto yystate0 + } +yyrule15: // [ \t]+ + { + l.tstart = l.i + goto yystate0 + } +yyrule16: // {D}+ + { + ts, err := strconv.ParseInt(yoloString(l.b[l.tstart:l.i]), 10, 64) + if err != nil { + l.err = err + return -1 + } + l.ts = &ts + goto yystate0 + } +yyrule17: // [\r\n]+ + { + l.nextMstart = l.i + return 1 + } +yyrule18: // \0 + { return 1 } @@ -392,5 +471,6 @@ yyrule14: // [^\n \t\r]+ goto yyabort // silence unused label error yyabort: // no lexem recognized + l.err = fmt.Errorf("no token found") return -1 } diff --git a/pkg/textparse/parse.go b/pkg/textparse/parse.go index dc6ef4ba5..b3cdd6fcd 100644 --- a/pkg/textparse/parse.go +++ b/pkg/textparse/parse.go @@ -30,11 +30,14 @@ type lexer struct { b []byte i int vstart int + tstart int err error val float64 + ts *int64 offsets []int mstart, mend int + nextMstart int } const eof = 0 @@ -70,7 +73,7 @@ func New(b []byte) *Parser { // more samples were read or an error occurred. func (p *Parser) Next() bool { switch p.l.Lex() { - case 0, -1: + case -1, eof: return false case 1: return true @@ -81,7 +84,7 @@ func (p *Parser) Next() bool { // At returns the bytes of the metric, the timestamp if set, and the value // of the current sample. func (p *Parser) At() ([]byte, *int64, float64) { - return p.l.b[p.l.mstart:p.l.mend], nil, p.l.val + return p.l.b[p.l.mstart:p.l.mend], p.l.ts, p.l.val } // Err returns the current error. diff --git a/pkg/textparse/parse_test.go b/pkg/textparse/parse_test.go index 06d5f9c69..bfb91aa30 100644 --- a/pkg/textparse/parse_test.go +++ b/pkg/textparse/parse_test.go @@ -36,11 +36,14 @@ go_gc_duration_seconds{quantile="0.5",a="b"} 8.3835e-05 go_gc_duration_seconds_count 99 # HELP go_goroutines Number of goroutines that currently exist. # TYPE go_goroutines gauge -go_goroutines 33` +go_goroutines 33 123123` + + int64p := func(x int64) *int64 { return &x } exp := []struct { lset labels.Labels m string + t *int64 v float64 }{ { @@ -62,6 +65,7 @@ go_goroutines 33` }, { m: `go_goroutines`, v: 33, + t: int64p(123123), lset: labels.FromStrings("__name__", "go_goroutines"), }, } @@ -72,11 +76,12 @@ go_goroutines 33` var res labels.Labels for p.Next() { - m, _, v := p.At() + m, ts, v := p.At() p.Metric(&res) require.Equal(t, exp[i].m, string(m)) + require.Equal(t, exp[i].t, ts) require.Equal(t, exp[i].v, v) require.Equal(t, exp[i].lset, res) @@ -85,6 +90,7 @@ go_goroutines 33` } require.NoError(t, p.Err()) + require.Equal(t, len(exp), i) }