2018-10-09 13:09:44 +00:00
// Copyright 2018 The Prometheus Authors
2018-10-04 16:57:47 +00:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2018-12-24 22:59:32 +00:00
//go:generate go get -u modernc.org/golex
2018-10-09 13:09:44 +00:00
//go:generate golex -o=openmetricslex.l.go openmetricslex.l
2018-10-04 16:57:47 +00:00
package textparse
import (
2024-10-10 11:01:13 +00:00
"bytes"
2022-06-27 19:29:19 +00:00
"errors"
2019-11-19 09:33:30 +00:00
"fmt"
2018-10-04 16:57:47 +00:00
"io"
"math"
2024-10-15 14:23:27 +00:00
"strconv"
2018-10-04 16:57:47 +00:00
"strings"
"unicode/utf8"
2024-10-10 11:01:13 +00:00
"github.com/cespare/xxhash/v2"
2023-11-22 14:39:21 +00:00
"github.com/prometheus/common/model"
2021-11-08 14:23:17 +00:00
"github.com/prometheus/prometheus/model/exemplar"
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 13:57:07 +00:00
"github.com/prometheus/prometheus/model/histogram"
2021-11-08 14:23:17 +00:00
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/value"
2018-10-04 16:57:47 +00:00
)
2018-10-09 13:09:44 +00:00
type openMetricsLexer struct {
2018-10-04 16:57:47 +00:00
b [ ] byte
i int
start int
err error
state int
}
// buf returns the buffer of the current token.
2018-10-09 13:09:44 +00:00
func ( l * openMetricsLexer ) buf ( ) [ ] byte {
2018-10-04 16:57:47 +00:00
return l . b [ l . start : l . i ]
}
2018-10-09 13:09:44 +00:00
// next advances the openMetricsLexer to the next character.
func ( l * openMetricsLexer ) next ( ) byte {
2018-10-04 16:57:47 +00:00
l . i ++
if l . i >= len ( l . b ) {
l . err = io . EOF
return byte ( tEOF )
}
// Lex struggles with null bytes. If we are in a label value or help string, where
// they are allowed, consume them here immediately.
for l . b [ l . i ] == 0 && ( l . state == sLValue || l . state == sMeta2 || l . state == sComment ) {
l . i ++
if l . i >= len ( l . b ) {
l . err = io . EOF
return byte ( tEOF )
}
}
return l . b [ l . i ]
}
2018-10-09 13:09:44 +00:00
func ( l * openMetricsLexer ) Error ( es string ) {
2018-10-04 16:57:47 +00:00
l . err = errors . New ( es )
}
2018-10-09 13:09:44 +00:00
// OpenMetricsParser parses samples from a byte slice of samples in the official
2018-10-04 16:57:47 +00:00
// OpenMetrics text exposition format.
2018-10-09 13:09:44 +00:00
// This is based on the working draft https://docs.google.com/document/u/1/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit
type OpenMetricsParser struct {
2024-10-10 11:01:13 +00:00
l * openMetricsLexer
builder labels . ScratchBuilder
series [ ] byte
mfNameLen int // length of metric family name to get from series.
text [ ] byte
mtype model . MetricType
val float64
ts int64
hasTS bool
start int
2024-02-15 19:25:12 +00:00
// offsets is a list of offsets into series that describe the positions
// of the metric name and label names and values for this series.
// p.offsets[0] is the start character of the metric name.
// p.offsets[1] is the end of the metric name.
// Subsequently, p.offsets is a pair of pair of offsets for the positions
// of the label name and value start and end characters.
2018-10-04 16:57:47 +00:00
offsets [ ] int
2019-11-19 09:33:30 +00:00
eOffsets [ ] int
exemplar [ ] byte
exemplarVal float64
exemplarTs int64
hasExemplarTs bool
2024-08-08 11:35:35 +00:00
2024-10-04 12:11:02 +00:00
// Created timestamp parsing state.
ct int64
ctHashSet uint64
2024-10-24 05:38:58 +00:00
// ignoreExemplar instructs the parser to not overwrite exemplars (to keep them while peeking ahead).
ignoreExemplar bool
2024-10-10 11:01:13 +00:00
// visitedMFName is the metric family name of the last visited metric when peeking ahead
2024-10-04 12:11:02 +00:00
// for _created series during the execution of the CreatedTimestamp method.
2024-10-10 11:01:13 +00:00
visitedMFName [ ] byte
skipCTSeries bool
2024-08-08 11:35:35 +00:00
}
type openMetricsParserOptions struct {
SkipCTSeries bool
2018-10-04 16:57:47 +00:00
}
2024-08-08 11:35:35 +00:00
type OpenMetricsOption func ( * openMetricsParserOptions )
// WithOMParserCTSeriesSkipped turns off exposing _created lines
// as series, which makes those only used for parsing created timestamp
// for `CreatedTimestamp` method purposes.
//
// It's recommended to use this option to avoid using _created lines for other
// purposes than created timestamp, but leave false by default for the
// best-effort compatibility.
func WithOMParserCTSeriesSkipped ( ) OpenMetricsOption {
return func ( o * openMetricsParserOptions ) {
o . SkipCTSeries = true
2023-03-25 13:31:24 +00:00
}
2018-10-04 16:57:47 +00:00
}
2024-08-08 11:35:35 +00:00
// NewOpenMetricsParser returns a new parser for the byte slice with option to skip CT series parsing.
func NewOpenMetricsParser ( b [ ] byte , st * labels . SymbolTable , opts ... OpenMetricsOption ) Parser {
options := & openMetricsParserOptions { }
for _ , opt := range opts {
opt ( options )
}
parser := & OpenMetricsParser {
l : & openMetricsLexer { b : b } ,
builder : labels . NewScratchBuilderWithSymbolTable ( st , 16 ) ,
skipCTSeries : options . SkipCTSeries ,
}
return parser
}
2018-10-04 16:57:47 +00:00
// Series returns the bytes of the series, the timestamp if set, and the value
// of the current sample.
2018-10-09 13:09:44 +00:00
func ( p * OpenMetricsParser ) Series ( ) ( [ ] byte , * int64 , float64 ) {
2018-10-04 16:57:47 +00:00
if p . hasTS {
2019-11-19 09:33:30 +00:00
ts := p . ts
return p . series , & ts , p . val
2018-10-04 16:57:47 +00:00
}
return p . series , nil , p . val
}
2023-01-10 23:30:55 +00:00
// Histogram returns (nil, nil, nil, nil) for now because OpenMetrics does not
// support sparse histograms yet.
2022-08-25 15:07:41 +00:00
func ( p * OpenMetricsParser ) Histogram ( ) ( [ ] byte , * int64 , * histogram . Histogram , * histogram . FloatHistogram ) {
return nil , nil , nil , nil
2021-06-29 21:45:23 +00:00
}
2018-10-04 16:57:47 +00:00
// Help returns the metric name and help text in the current entry.
// Must only be called after Next returned a help entry.
// The returned byte slices become invalid after the next call to Next.
2018-10-09 13:09:44 +00:00
func ( p * OpenMetricsParser ) Help ( ) ( [ ] byte , [ ] byte ) {
2018-10-04 16:57:47 +00:00
m := p . l . b [ p . offsets [ 0 ] : p . offsets [ 1 ] ]
// Replacer causes allocations. Replace only when necessary.
if strings . IndexByte ( yoloString ( p . text ) , byte ( '\\' ) ) >= 0 {
// OpenMetrics always uses the Prometheus format label value escaping.
return m , [ ] byte ( lvalReplacer . Replace ( string ( p . text ) ) )
}
return m , p . text
}
// Type returns the metric name and type in the current entry.
// Must only be called after Next returned a type entry.
// The returned byte slices become invalid after the next call to Next.
2023-12-12 12:14:36 +00:00
func ( p * OpenMetricsParser ) Type ( ) ( [ ] byte , model . MetricType ) {
2018-10-04 16:57:47 +00:00
return p . l . b [ p . offsets [ 0 ] : p . offsets [ 1 ] ] , p . mtype
}
// Unit returns the metric name and unit in the current entry.
// Must only be called after Next returned a unit entry.
// The returned byte slices become invalid after the next call to Next.
2018-10-09 13:09:44 +00:00
func ( p * OpenMetricsParser ) Unit ( ) ( [ ] byte , [ ] byte ) {
2018-10-04 16:57:47 +00:00
return p . l . b [ p . offsets [ 0 ] : p . offsets [ 1 ] ] , p . text
}
// Comment returns the text of the current comment.
// Must only be called after Next returned a comment entry.
// The returned byte slice becomes invalid after the next call to Next.
2018-10-09 13:09:44 +00:00
func ( p * OpenMetricsParser ) Comment ( ) [ ] byte {
2018-10-04 16:57:47 +00:00
return p . text
}
// Metric writes the labels of the current sample into the passed labels.
// It returns the string from which the metric was parsed.
2018-10-09 13:09:44 +00:00
func ( p * OpenMetricsParser ) Metric ( l * labels . Labels ) string {
2022-03-09 22:13:50 +00:00
// Copy the buffer to a string: this is only necessary for the return value.
2018-10-04 16:57:47 +00:00
s := string ( p . series )
2022-03-09 22:13:50 +00:00
p . builder . Reset ( )
2024-02-15 19:25:12 +00:00
metricName := unreplace ( s [ p . offsets [ 0 ] - p . start : p . offsets [ 1 ] - p . start ] )
p . builder . Add ( labels . MetricName , metricName )
2018-10-04 16:57:47 +00:00
2024-02-15 19:25:12 +00:00
for i := 2 ; i < len ( p . offsets ) ; i += 4 {
2018-10-04 16:57:47 +00:00
a := p . offsets [ i ] - p . start
b := p . offsets [ i + 1 ] - p . start
2024-02-15 19:25:12 +00:00
label := unreplace ( s [ a : b ] )
2018-10-04 16:57:47 +00:00
c := p . offsets [ i + 2 ] - p . start
d := p . offsets [ i + 3 ] - p . start
2024-10-15 14:23:27 +00:00
value := normalizeFloatsInLabelValues ( p . mtype , label , unreplace ( s [ c : d ] ) )
2018-10-04 16:57:47 +00:00
2024-02-15 19:25:12 +00:00
p . builder . Add ( label , value )
2018-10-04 16:57:47 +00:00
}
2022-03-09 22:13:50 +00:00
p . builder . Sort ( )
* l = p . builder . Labels ( )
2018-10-04 16:57:47 +00:00
return s
}
2023-07-13 12:16:10 +00:00
// Exemplar writes the exemplar of the current sample into the passed exemplar.
// It returns whether an exemplar exists. As OpenMetrics only ever has one
// exemplar per sample, every call after the first (for the same sample) will
// always return false.
2019-11-19 09:33:30 +00:00
func ( p * OpenMetricsParser ) Exemplar ( e * exemplar . Exemplar ) bool {
if len ( p . exemplar ) == 0 {
return false
}
// Allocate the full immutable string immediately, so we just
// have to create references on it below.
s := string ( p . exemplar )
e . Value = p . exemplarVal
if p . hasExemplarTs {
e . HasTs = true
e . Ts = p . exemplarTs
}
2022-03-09 22:13:50 +00:00
p . builder . Reset ( )
2019-11-19 09:33:30 +00:00
for i := 0 ; i < len ( p . eOffsets ) ; i += 4 {
a := p . eOffsets [ i ] - p . start
b := p . eOffsets [ i + 1 ] - p . start
c := p . eOffsets [ i + 2 ] - p . start
d := p . eOffsets [ i + 3 ] - p . start
2022-03-09 22:13:50 +00:00
p . builder . Add ( s [ a : b ] , s [ c : d ] )
2019-11-19 09:33:30 +00:00
}
2022-03-09 22:13:50 +00:00
p . builder . Sort ( )
e . Labels = p . builder . Labels ( )
2019-11-19 09:33:30 +00:00
2023-07-13 12:16:10 +00:00
// Wipe exemplar so that future calls return false.
p . exemplar = p . exemplar [ : 0 ]
2019-11-19 09:33:30 +00:00
return true
}
2024-08-08 11:35:35 +00:00
// CreatedTimestamp returns the created timestamp for a current Metric if exists or nil.
// NOTE(Maniktherana): Might use additional CPU/mem resources due to deep copy of parser required for peeking given 1.0 OM specification on _created series.
2023-12-11 08:43:42 +00:00
func ( p * OpenMetricsParser ) CreatedTimestamp ( ) * int64 {
2024-09-20 16:26:19 +00:00
if ! typeRequiresCT ( p . mtype ) {
2024-08-08 11:35:35 +00:00
// Not a CT supported metric type, fast path.
2024-10-10 11:01:13 +00:00
p . ctHashSet = 0 // Use ctHashSet as a single way of telling "empty cache"
2024-08-08 11:35:35 +00:00
return nil
}
var (
2024-10-10 11:01:13 +00:00
buf [ ] byte
currName [ ] byte
2024-08-08 11:35:35 +00:00
)
2024-10-10 11:01:13 +00:00
if len ( p . series ) > 1 && p . series [ 0 ] == '{' && p . series [ 1 ] == '"' {
// special case for UTF-8 encoded metric family names.
currName = p . series [ p . offsets [ 0 ] - p . start : p . mfNameLen + 2 ]
} else {
currName = p . series [ p . offsets [ 0 ] - p . start : p . mfNameLen ]
}
currHash := p . seriesHash ( & buf , currName )
// Check cache, perhaps we fetched something already.
if currHash == p . ctHashSet && p . ct > 0 {
2024-10-04 12:11:02 +00:00
return & p . ct
}
// Create a new lexer to reset the parser once this function is done executing.
resetLexer := & openMetricsLexer {
b : p . l . b ,
i : p . l . i ,
start : p . l . start ,
err : p . l . err ,
state : p . l . state ,
}
p . skipCTSeries = false
2024-10-24 05:38:58 +00:00
p . ignoreExemplar = true
savedStart := p . start
defer func ( ) {
p . ignoreExemplar = false
p . start = savedStart
p . l = resetLexer
} ( )
2024-08-08 11:35:35 +00:00
for {
2024-10-04 12:11:02 +00:00
eType , err := p . Next ( )
2024-08-08 11:35:35 +00:00
if err != nil {
2024-10-04 12:11:02 +00:00
// This means p.Next() will give error too later on, so def no CT line found.
2024-08-08 11:35:35 +00:00
// This might result in partial scrape with wrong/missing CT, but only
// spec improvement would help.
2024-10-04 12:11:02 +00:00
// TODO: Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
2024-10-24 05:38:58 +00:00
p . resetCTParseValues ( )
2024-08-08 11:35:35 +00:00
return nil
}
if eType != EntrySeries {
// Assume we hit different family, no CT line found.
2024-10-24 05:38:58 +00:00
p . resetCTParseValues ( )
2024-08-08 11:35:35 +00:00
return nil
}
2024-10-10 11:01:13 +00:00
peekedName := p . series [ p . offsets [ 0 ] - p . start : p . offsets [ 1 ] - p . start ]
if len ( peekedName ) < 8 || string ( peekedName [ len ( peekedName ) - 8 : ] ) != "_created" {
2024-08-08 11:35:35 +00:00
// Not a CT line, search more.
continue
}
2024-10-10 11:01:13 +00:00
// Remove _created suffix.
peekedHash := p . seriesHash ( & buf , peekedName [ : len ( peekedName ) - 8 ] )
if peekedHash != currHash {
2024-10-04 12:11:02 +00:00
// Found CT line for a different series, for our series no CT.
2024-10-24 05:38:58 +00:00
p . resetCTParseValues ( )
2024-08-08 11:35:35 +00:00
return nil
}
2024-10-02 10:52:03 +00:00
// All timestamps in OpenMetrics are Unix Epoch in seconds. Convert to milliseconds.
// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#timestamps
2024-10-04 12:11:02 +00:00
ct := int64 ( p . val * 1000.0 )
2024-10-24 05:38:58 +00:00
p . setCTParseValues ( ct , currHash , currName , true )
2024-08-08 11:35:35 +00:00
return & ct
}
}
2024-10-10 11:01:13 +00:00
var (
leBytes = [ ] byte { 108 , 101 }
quantileBytes = [ ] byte { 113 , 117 , 97 , 110 , 116 , 105 , 108 , 101 }
)
// seriesHash generates a hash based on the metric family name and the offsets
// of label names and values from the parsed OpenMetrics data. It skips quantile
// and le labels for summaries and histograms respectively.
func ( p * OpenMetricsParser ) seriesHash ( offsetsArr * [ ] byte , metricFamilyName [ ] byte ) uint64 {
// Iterate through p.offsets to find the label names and values.
for i := 2 ; i < len ( p . offsets ) ; i += 4 {
lStart := p . offsets [ i ] - p . start
lEnd := p . offsets [ i + 1 ] - p . start
label := p . series [ lStart : lEnd ]
// Skip quantile and le labels for summaries and histograms.
if p . mtype == model . MetricTypeSummary && bytes . Equal ( label , quantileBytes ) {
continue
}
if p . mtype == model . MetricTypeHistogram && bytes . Equal ( label , leBytes ) {
continue
}
* offsetsArr = append ( * offsetsArr , p . series [ lStart : lEnd ] ... )
vStart := p . offsets [ i + 2 ] - p . start
vEnd := p . offsets [ i + 3 ] - p . start
* offsetsArr = append ( * offsetsArr , p . series [ vStart : vEnd ] ... )
}
* offsetsArr = append ( * offsetsArr , metricFamilyName ... )
hashedOffsets := xxhash . Sum64 ( * offsetsArr )
// Reset the offsets array for later reuse.
* offsetsArr = ( * offsetsArr ) [ : 0 ]
return hashedOffsets
}
2024-10-04 12:11:02 +00:00
// setCTParseValues sets the parser to the state after CreatedTimestamp method was called and CT was found.
// This is useful to prevent re-parsing the same series again and early return the CT value.
2024-10-24 05:38:58 +00:00
func ( p * OpenMetricsParser ) setCTParseValues ( ct int64 , ctHashSet uint64 , mfName [ ] byte , skipCTSeries bool ) {
2024-10-04 12:11:02 +00:00
p . ct = ct
p . ctHashSet = ctHashSet
2024-10-10 11:01:13 +00:00
p . visitedMFName = mfName
p . skipCTSeries = skipCTSeries // Do we need to set it?
2024-10-04 12:11:02 +00:00
}
2024-11-19 04:02:10 +00:00
// resetCTParseValues resets the parser to the state before CreatedTimestamp method was called.
2024-10-24 05:38:58 +00:00
func ( p * OpenMetricsParser ) resetCTParseValues ( ) {
2024-10-04 12:11:02 +00:00
p . ctHashSet = 0
p . skipCTSeries = true
}
2024-09-20 16:26:19 +00:00
// typeRequiresCT returns true if the metric type requires a _created timestamp.
func typeRequiresCT ( t model . MetricType ) bool {
2024-08-08 11:35:35 +00:00
switch t {
case model . MetricTypeCounter , model . MetricTypeSummary , model . MetricTypeHistogram :
return true
default :
return false
}
}
2018-10-09 13:09:44 +00:00
// nextToken returns the next token from the openMetricsLexer.
func ( p * OpenMetricsParser ) nextToken ( ) token {
2018-10-04 16:57:47 +00:00
tok := p . l . Lex ( )
return tok
}
2023-02-03 15:50:15 +00:00
func ( p * OpenMetricsParser ) parseError ( exp string , got token ) error {
e := p . l . i + 1
if len ( p . l . b ) < e {
e = len ( p . l . b )
}
return fmt . Errorf ( "%s, got %q (%q) while parsing: %q" , exp , p . l . b [ p . l . start : e ] , got , p . l . b [ p . start : e ] )
2022-12-07 13:32:03 +00:00
}
2024-02-29 15:40:53 +00:00
// Next advances the parser to the next sample.
// It returns (EntryInvalid, io.EOF) if no samples were read.
2018-10-09 13:09:44 +00:00
func ( p * OpenMetricsParser ) Next ( ) ( Entry , error ) {
2018-10-04 16:57:47 +00:00
var err error
p . start = p . l . i
p . offsets = p . offsets [ : 0 ]
2024-10-24 05:38:58 +00:00
if ! p . ignoreExemplar {
p . eOffsets = p . eOffsets [ : 0 ]
p . exemplar = p . exemplar [ : 0 ]
p . exemplarVal = 0
p . hasExemplarTs = false
}
2018-10-04 16:57:47 +00:00
2020-02-07 07:33:26 +00:00
switch t := p . nextToken ( ) ; t {
2020-03-23 14:47:11 +00:00
case tEOFWord :
2018-10-04 16:57:47 +00:00
if t := p . nextToken ( ) ; t != tEOF {
2019-03-25 23:01:12 +00:00
return EntryInvalid , errors . New ( "unexpected data after # EOF" )
2018-10-04 16:57:47 +00:00
}
return EntryInvalid , io . EOF
case tEOF :
2020-02-07 07:33:26 +00:00
return EntryInvalid , errors . New ( "data does not end with # EOF" )
2018-10-04 16:57:47 +00:00
case tHelp , tType , tUnit :
2022-03-08 13:04:11 +00:00
switch t2 := p . nextToken ( ) ; t2 {
2018-10-04 16:57:47 +00:00
case tMName :
2024-02-15 19:25:12 +00:00
mStart := p . l . start
mEnd := p . l . i
if p . l . b [ mStart ] == '"' && p . l . b [ mEnd - 1 ] == '"' {
mStart ++
mEnd --
}
2024-10-10 11:01:13 +00:00
p . mfNameLen = mEnd - mStart
2024-02-15 19:25:12 +00:00
p . offsets = append ( p . offsets , mStart , mEnd )
2018-10-04 16:57:47 +00:00
default :
2023-02-03 15:50:15 +00:00
return EntryInvalid , p . parseError ( "expected metric name after " + t . String ( ) , t2 )
2018-10-04 16:57:47 +00:00
}
2022-03-08 13:04:11 +00:00
switch t2 := p . nextToken ( ) ; t2 {
2018-10-04 16:57:47 +00:00
case tText :
if len ( p . l . buf ( ) ) > 1 {
p . text = p . l . buf ( ) [ 1 : len ( p . l . buf ( ) ) - 1 ]
} else {
p . text = [ ] byte { }
}
default :
2022-03-08 13:04:11 +00:00
return EntryInvalid , fmt . Errorf ( "expected text in %s" , t . String ( ) )
2018-10-04 16:57:47 +00:00
}
switch t {
case tType :
switch s := yoloString ( p . text ) ; s {
case "counter" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeCounter
2018-10-04 16:57:47 +00:00
case "gauge" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeGauge
2018-10-04 16:57:47 +00:00
case "histogram" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeHistogram
2018-10-04 16:57:47 +00:00
case "gaugehistogram" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeGaugeHistogram
2018-10-04 16:57:47 +00:00
case "summary" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeSummary
2018-10-04 16:57:47 +00:00
case "info" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeInfo
2018-10-04 16:57:47 +00:00
case "stateset" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeStateset
2018-10-04 16:57:47 +00:00
case "unknown" :
2023-11-22 14:39:21 +00:00
p . mtype = model . MetricTypeUnknown
2018-10-04 16:57:47 +00:00
default :
2022-06-27 19:29:19 +00:00
return EntryInvalid , fmt . Errorf ( "invalid metric type %q" , s )
2018-10-04 16:57:47 +00:00
}
case tHelp :
if ! utf8 . Valid ( p . text ) {
2023-02-03 15:50:15 +00:00
return EntryInvalid , fmt . Errorf ( "help text %q is not a valid utf8 string" , p . text )
2018-10-04 16:57:47 +00:00
}
}
switch t {
case tHelp :
return EntryHelp , nil
case tType :
return EntryType , nil
case tUnit :
m := yoloString ( p . l . b [ p . offsets [ 0 ] : p . offsets [ 1 ] ] )
u := yoloString ( p . text )
if len ( u ) > 0 {
if ! strings . HasSuffix ( m , u ) || len ( m ) < len ( u ) + 1 || p . l . b [ p . offsets [ 1 ] - len ( u ) - 1 ] != '_' {
2023-02-03 15:50:15 +00:00
return EntryInvalid , fmt . Errorf ( "unit %q not a suffix of metric %q" , u , m )
2018-10-04 16:57:47 +00:00
}
}
return EntryUnit , nil
}
2024-02-15 19:25:12 +00:00
case tBraceOpen :
// We found a brace, so make room for the eventual metric name. If these
// values aren't updated, then the metric name was not set inside the
// braces and we can return an error.
if len ( p . offsets ) == 0 {
p . offsets = [ ] int { - 1 , - 1 }
}
if p . offsets , err = p . parseLVals ( p . offsets , false ) ; err != nil {
return EntryInvalid , err
}
p . series = p . l . b [ p . start : p . l . i ]
2024-08-08 11:35:35 +00:00
if err := p . parseSeriesEndOfLine ( p . nextToken ( ) ) ; err != nil {
return EntryInvalid , err
}
if p . skipCTSeries && p . isCreatedSeries ( ) {
return p . Next ( )
}
return EntrySeries , nil
2018-10-04 16:57:47 +00:00
case tMName :
2024-02-15 19:25:12 +00:00
p . offsets = append ( p . offsets , p . start , p . l . i )
2018-10-04 16:57:47 +00:00
p . series = p . l . b [ p . start : p . l . i ]
t2 := p . nextToken ( )
if t2 == tBraceOpen {
2024-02-15 19:25:12 +00:00
p . offsets , err = p . parseLVals ( p . offsets , false )
2019-11-19 09:33:30 +00:00
if err != nil {
2018-10-04 16:57:47 +00:00
return EntryInvalid , err
}
p . series = p . l . b [ p . start : p . l . i ]
t2 = p . nextToken ( )
}
2024-08-08 11:35:35 +00:00
if err := p . parseSeriesEndOfLine ( t2 ) ; err != nil {
return EntryInvalid , err
}
if p . skipCTSeries && p . isCreatedSeries ( ) {
return p . Next ( )
}
return EntrySeries , nil
2018-10-04 16:57:47 +00:00
default :
2023-02-03 15:50:15 +00:00
err = p . parseError ( "expected a valid start token" , t )
2018-10-04 16:57:47 +00:00
}
return EntryInvalid , err
}
2019-11-19 09:33:30 +00:00
func ( p * OpenMetricsParser ) parseComment ( ) error {
2021-09-08 08:09:21 +00:00
var err error
2024-10-24 05:38:58 +00:00
if p . ignoreExemplar {
for t := p . nextToken ( ) ; t != tLinebreak ; t = p . nextToken ( ) {
if t == tEOF {
return errors . New ( "data does not end with # EOF" )
}
}
return nil
}
2019-11-19 09:33:30 +00:00
// Parse the labels.
2024-02-15 19:25:12 +00:00
p . eOffsets , err = p . parseLVals ( p . eOffsets , true )
2019-11-19 09:33:30 +00:00
if err != nil {
return err
}
p . exemplar = p . l . b [ p . start : p . l . i ]
// Get the value.
p . exemplarVal , err = p . getFloatValue ( p . nextToken ( ) , "exemplar labels" )
if err != nil {
return err
}
// Read the optional timestamp.
p . hasExemplarTs = false
switch t2 := p . nextToken ( ) ; t2 {
case tEOF :
2019-12-24 08:48:28 +00:00
return errors . New ( "data does not end with # EOF" )
2019-11-19 09:33:30 +00:00
case tLinebreak :
break
case tTimestamp :
p . hasExemplarTs = true
var ts float64
// A float is enough to hold what we need for millisecond resolution.
if ts , err = parseFloat ( yoloString ( p . l . buf ( ) [ 1 : ] ) ) ; err != nil {
2023-09-29 20:50:30 +00:00
return fmt . Errorf ( "%w while parsing: %q" , err , p . l . b [ p . start : p . l . i ] )
2019-11-19 09:33:30 +00:00
}
2021-06-29 00:54:50 +00:00
if math . IsNaN ( ts ) || math . IsInf ( ts , 0 ) {
2023-02-03 15:50:15 +00:00
return fmt . Errorf ( "invalid exemplar timestamp %f" , ts )
2021-06-29 00:54:50 +00:00
}
2019-11-19 09:33:30 +00:00
p . exemplarTs = int64 ( ts * 1000 )
switch t3 := p . nextToken ( ) ; t3 {
case tLinebreak :
default :
2023-02-03 15:50:15 +00:00
return p . parseError ( "expected next entry after exemplar timestamp" , t3 )
2019-11-19 09:33:30 +00:00
}
default :
2023-02-03 15:50:15 +00:00
return p . parseError ( "expected timestamp or comment" , t2 )
2019-11-19 09:33:30 +00:00
}
return nil
}
2024-02-15 19:25:12 +00:00
func ( p * OpenMetricsParser ) parseLVals ( offsets [ ] int , isExemplar bool ) ( [ ] int , error ) {
t := p . nextToken ( )
2018-10-04 16:57:47 +00:00
for {
2024-02-15 19:25:12 +00:00
curTStart := p . l . start
curTI := p . l . i
2018-10-04 16:57:47 +00:00
switch t {
case tBraceClose :
2019-11-19 09:33:30 +00:00
return offsets , nil
2024-02-15 19:25:12 +00:00
case tLName :
case tQString :
default :
return nil , p . parseError ( "expected label name" , t )
}
t = p . nextToken ( )
// A quoted string followed by a comma or brace is a metric name. Set the
// offsets and continue processing. If this is an exemplar, this format
// is not allowed.
if t == tComma || t == tBraceClose {
if isExemplar {
2023-02-03 15:50:15 +00:00
return nil , p . parseError ( "expected label name" , t )
2018-10-04 16:57:47 +00:00
}
2024-02-15 19:25:12 +00:00
if offsets [ 0 ] != - 1 || offsets [ 1 ] != - 1 {
return nil , fmt . Errorf ( "metric name already set while parsing: %q" , p . l . b [ p . start : p . l . i ] )
2018-10-04 16:57:47 +00:00
}
2024-02-15 19:25:12 +00:00
offsets [ 0 ] = curTStart + 1
offsets [ 1 ] = curTI - 1
if t == tBraceClose {
return offsets , nil
2018-10-04 16:57:47 +00:00
}
2024-02-15 19:25:12 +00:00
t = p . nextToken ( )
continue
2018-10-04 16:57:47 +00:00
}
2024-02-15 19:25:12 +00:00
// We have a label name, and it might be quoted.
if p . l . b [ curTStart ] == '"' {
curTStart ++
curTI --
}
offsets = append ( offsets , curTStart , curTI )
2018-10-04 16:57:47 +00:00
2024-02-15 19:25:12 +00:00
if t != tEqual {
2023-02-03 15:50:15 +00:00
return nil , p . parseError ( "expected equal" , t )
2018-10-04 16:57:47 +00:00
}
if t := p . nextToken ( ) ; t != tLValue {
2023-02-03 15:50:15 +00:00
return nil , p . parseError ( "expected label value" , t )
2018-10-04 16:57:47 +00:00
}
if ! utf8 . Valid ( p . l . buf ( ) ) {
2023-02-03 15:50:15 +00:00
return nil , fmt . Errorf ( "invalid UTF-8 label value: %q" , p . l . buf ( ) )
2018-10-04 16:57:47 +00:00
}
2018-10-09 13:09:44 +00:00
// The openMetricsLexer ensures the value string is quoted. Strip first
2018-10-04 16:57:47 +00:00
// and last character.
2019-11-19 09:33:30 +00:00
offsets = append ( offsets , p . l . start + 1 , p . l . i - 1 )
2024-02-15 19:25:12 +00:00
// Free trailing commas are allowed.
t = p . nextToken ( )
if t == tComma {
t = p . nextToken ( )
} else if t != tBraceClose {
return nil , p . parseError ( "expected comma or brace close" , t )
}
}
}
2024-08-08 11:35:35 +00:00
// isCreatedSeries returns true if the current series is a _created series.
func ( p * OpenMetricsParser ) isCreatedSeries ( ) bool {
2024-10-14 09:18:15 +00:00
metricName := p . series [ p . offsets [ 0 ] - p . start : p . offsets [ 1 ] - p . start ]
// check length so the metric is longer than len("_created")
if typeRequiresCT ( p . mtype ) && len ( metricName ) >= 8 && string ( metricName [ len ( metricName ) - 8 : ] ) == "_created" {
2024-08-08 11:35:35 +00:00
return true
}
return false
}
// parseSeriesEndOfLine parses the series end of the line (value, optional
// timestamp, commentary, etc.) after the metric name and labels.
// It starts parsing with the provided token.
func ( p * OpenMetricsParser ) parseSeriesEndOfLine ( t token ) error {
2024-02-15 19:25:12 +00:00
if p . offsets [ 0 ] == - 1 {
2024-08-08 11:35:35 +00:00
return fmt . Errorf ( "metric name not set while parsing: %q" , p . l . b [ p . start : p . l . i ] )
2024-02-15 19:25:12 +00:00
}
var err error
p . val , err = p . getFloatValue ( t , "metric" )
if err != nil {
2024-08-08 11:35:35 +00:00
return err
2024-02-15 19:25:12 +00:00
}
p . hasTS = false
switch t2 := p . nextToken ( ) ; t2 {
case tEOF :
2024-08-08 11:35:35 +00:00
return errors . New ( "data does not end with # EOF" )
2024-02-15 19:25:12 +00:00
case tLinebreak :
break
case tComment :
if err := p . parseComment ( ) ; err != nil {
2024-08-08 11:35:35 +00:00
return err
2024-02-15 19:25:12 +00:00
}
case tTimestamp :
p . hasTS = true
var ts float64
// A float is enough to hold what we need for millisecond resolution.
if ts , err = parseFloat ( yoloString ( p . l . buf ( ) [ 1 : ] ) ) ; err != nil {
2024-08-08 11:35:35 +00:00
return fmt . Errorf ( "%w while parsing: %q" , err , p . l . b [ p . start : p . l . i ] )
2024-02-15 19:25:12 +00:00
}
if math . IsNaN ( ts ) || math . IsInf ( ts , 0 ) {
2024-08-08 11:35:35 +00:00
return fmt . Errorf ( "invalid timestamp %f" , ts )
2024-02-15 19:25:12 +00:00
}
p . ts = int64 ( ts * 1000 )
switch t3 := p . nextToken ( ) ; t3 {
case tLinebreak :
case tComment :
if err := p . parseComment ( ) ; err != nil {
2024-08-08 11:35:35 +00:00
return err
2024-02-15 19:25:12 +00:00
}
default :
2024-08-08 11:35:35 +00:00
return p . parseError ( "expected next entry after timestamp" , t3 )
2024-02-15 19:25:12 +00:00
}
2019-11-19 09:33:30 +00:00
}
2024-08-08 11:35:35 +00:00
return nil
2019-11-19 09:33:30 +00:00
}
func ( p * OpenMetricsParser ) getFloatValue ( t token , after string ) ( float64 , error ) {
if t != tValue {
2023-02-03 15:50:15 +00:00
return 0 , p . parseError ( fmt . Sprintf ( "expected value after %v" , after ) , t )
2019-11-19 09:33:30 +00:00
}
val , err := parseFloat ( yoloString ( p . l . buf ( ) [ 1 : ] ) )
if err != nil {
2023-09-29 20:50:30 +00:00
return 0 , fmt . Errorf ( "%w while parsing: %q" , err , p . l . b [ p . start : p . l . i ] )
2019-11-19 09:33:30 +00:00
}
// Ensure canonical NaN value.
if math . IsNaN ( p . exemplarVal ) {
val = math . Float64frombits ( value . NormalNaN )
}
return val , nil
}
2024-10-15 14:23:27 +00:00
// normalizeFloatsInLabelValues ensures that values of the "le" labels of classic histograms and "quantile" labels
// of summaries follow OpenMetrics formatting rules.
func normalizeFloatsInLabelValues ( t model . MetricType , l , v string ) string {
if ( t == model . MetricTypeSummary && l == model . QuantileLabel ) || ( t == model . MetricTypeHistogram && l == model . BucketLabel ) {
f , err := strconv . ParseFloat ( v , 64 )
if err == nil {
return formatOpenMetricsFloat ( f )
}
}
return v
}