From 92a3eeac55974f69afab50f838fac6df6dfb77fe Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 8 Sep 2021 09:09:21 +0100 Subject: [PATCH] Create less garbage when parsing metrics (#9299) * Refactor: extract function to make scrapeLoop for testing Signed-off-by: Bryan Boreham * Add benchmarks for ScrapeLoopAppend For Prometheus and OpenMetrics Signed-off-by: Bryan Boreham * Create less garbage when parsing metrics Exemplar escapes to heap due to being passed through text-parser interface, but we can reduce the impact by hoisting it out of the loop and resetting it after every use. (Note the cost was paid on every line even when exemplars were disabled) Signed-off-by: Bryan Boreham * Create less garbage when parsing OpenMetrics After calling parseLVals() we always append the return value, so pass in what we want to append it to and save garbage. Signed-off-by: Bryan Boreham --- pkg/textparse/openmetricsparse.go | 10 +++--- scrape/scrape.go | 3 +- scrape/scrape_test.go | 52 +++++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/pkg/textparse/openmetricsparse.go b/pkg/textparse/openmetricsparse.go index e3c09e97e..565efd359 100644 --- a/pkg/textparse/openmetricsparse.go +++ b/pkg/textparse/openmetricsparse.go @@ -306,11 +306,10 @@ func (p *OpenMetricsParser) Next() (Entry, error) { t2 := p.nextToken() if t2 == tBraceOpen { - offsets, err := p.parseLVals() + p.offsets, err = p.parseLVals(p.offsets) if err != nil { return EntryInvalid, err } - p.offsets = append(p.offsets, offsets...) p.series = p.l.b[p.start:p.l.i] t2 = p.nextToken() } @@ -367,12 +366,12 @@ func (p *OpenMetricsParser) parseComment() error { return err } + var err error // Parse the labels. - offsets, err := p.parseLVals() + p.eOffsets, err = p.parseLVals(p.eOffsets) if err != nil { return err } - p.eOffsets = append(p.eOffsets, offsets...) p.exemplar = p.l.b[p.start:p.l.i] // Get the value. @@ -410,8 +409,7 @@ func (p *OpenMetricsParser) parseComment() error { return nil } -func (p *OpenMetricsParser) parseLVals() ([]int, error) { - var offsets []int +func (p *OpenMetricsParser) parseLVals(offsets []int) ([]int, error) { first := true for { t := p.nextToken() diff --git a/scrape/scrape.go b/scrape/scrape.go index 5344e663a..ad0d632a3 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -1390,6 +1390,7 @@ func (sl *scrapeLoop) append(app storage.Appender, b []byte, contentType string, defTime = timestamp.FromTime(ts) appErrs = appendErrors{} sampleLimitErr error + e exemplar.Exemplar // escapes to heap so hoisted out of loop ) defer func() { @@ -1406,7 +1407,6 @@ loop: var ( et textparse.Entry sampleAdded bool - e exemplar.Exemplar ) if et, err = p.Next(); err != nil { if err == io.EOF { @@ -1513,6 +1513,7 @@ loop: // Since exemplar storage is still experimental, we don't fail the scrape on ingestion errors. level.Debug(sl.l).Log("msg", "Error while adding exemplar in AddExemplar", "exemplar", fmt.Sprintf("%+v", e), "err", exemplarErr) } + e = exemplar.Exemplar{} // reset for next time round loop } } diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index fc2b51ab7..9bff279c3 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -929,10 +929,10 @@ test_metric 1 require.Equal(t, "", md.Unit) } -func TestScrapeLoopSeriesAdded(t *testing.T) { +func simpleTestScrapeLoop(t testing.TB) (context.Context, *scrapeLoop) { // Need a full storage for correct Add/AddFast semantics. s := teststorage.New(t) - defer s.Close() + t.Cleanup(func() { s.Close() }) ctx, cancel := context.WithCancel(context.Background()) sl := newScrapeLoop(ctx, @@ -950,7 +950,13 @@ func TestScrapeLoopSeriesAdded(t *testing.T) { 0, false, ) - defer cancel() + t.Cleanup(func() { cancel() }) + + return ctx, sl +} + +func TestScrapeLoopSeriesAdded(t *testing.T) { + ctx, sl := simpleTestScrapeLoop(t) slApp := sl.appender(ctx) total, added, seriesAdded, err := sl.append(slApp, []byte("test_metric 1\n"), "", time.Time{}) @@ -969,6 +975,46 @@ func TestScrapeLoopSeriesAdded(t *testing.T) { require.Equal(t, 0, seriesAdded) } +func makeTestMetrics(n int) []byte { + // Construct a metrics string to parse + sb := bytes.Buffer{} + for i := 0; i < n; i++ { + fmt.Fprintf(&sb, "# TYPE metric_a gauge\n") + fmt.Fprintf(&sb, "# HELP metric_a help text\n") + fmt.Fprintf(&sb, "metric_a{foo=\"%d\",bar=\"%d\"} 1\n", i, i*100) + } + return sb.Bytes() +} + +func BenchmarkScrapeLoopAppend(b *testing.B) { + ctx, sl := simpleTestScrapeLoop(b) + + slApp := sl.appender(ctx) + metrics := makeTestMetrics(100) + ts := time.Time{} + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + ts = ts.Add(time.Second) + _, _, _, _ = sl.append(slApp, metrics, "", ts) + } +} +func BenchmarkScrapeLoopAppendOM(b *testing.B) { + ctx, sl := simpleTestScrapeLoop(b) + + slApp := sl.appender(ctx) + metrics := makeTestMetrics(100) + ts := time.Time{} + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + ts = ts.Add(time.Second) + _, _, _, _ = sl.append(slApp, metrics, "application/openmetrics-text", ts) + } +} + func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) { appender := &collectResultAppender{} var (