From 3069bd39961b695081ef6c9aab04165fc4a2a394 Mon Sep 17 00:00:00 2001
From: Goutham Veeramachaneni <goutham@boomerangcommerce.com>
Date: Tue, 4 Jul 2017 11:24:13 +0200
Subject: [PATCH] Handle scrapes with OutOfBounds metrics better

fixes #2894

Signed-off-by: Goutham Veeramachaneni <goutham@boomerangcommerce.com>
---
 retrieval/scrape.go  | 22 +++++++++++++++++-----
 storage/interface.go |  2 ++
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/retrieval/scrape.go b/retrieval/scrape.go
index 606638b5f..3ef064c23 100644
--- a/retrieval/scrape.go
+++ b/retrieval/scrape.go
@@ -722,11 +722,12 @@ func (s samples) Less(i, j int) bool {
 
 func (sl *scrapeLoop) append(b []byte, ts time.Time) (total, added int, err error) {
 	var (
-		app           = sl.appender()
-		p             = textparse.New(b)
-		defTime       = timestamp.FromTime(ts)
-		numOutOfOrder = 0
-		numDuplicates = 0
+		app            = sl.appender()
+		p              = textparse.New(b)
+		defTime        = timestamp.FromTime(ts)
+		numOutOfOrder  = 0
+		numDuplicates  = 0
+		numOutOfBounds = 0
 	)
 	var sampleLimitErr error
 
@@ -761,6 +762,10 @@ loop:
 				numDuplicates++
 				sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
 				continue
+			case storage.ErrOutOfBounds:
+				numOutOfBounds++
+				sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
+				continue
 			case errSampleLimit:
 				// Keep on parsing output if we hit the limit, so we report the correct
 				// total number of samples scraped.
@@ -804,6 +809,10 @@ loop:
 				numDuplicates++
 				sl.l.With("timeseries", string(met)).Debug("Duplicate sample for timestamp")
 				continue
+			case storage.ErrOutOfBounds:
+				numOutOfBounds++
+				sl.l.With("timeseries", string(met)).Debug("Out of bounds metric")
+				continue
 			case errSampleLimit:
 				sampleLimitErr = err
 				added++
@@ -832,6 +841,9 @@ loop:
 	if numDuplicates > 0 {
 		sl.l.With("numDropped", numDuplicates).Warn("Error on ingesting samples with different value but same timestamp")
 	}
+	if numOutOfBounds > 0 {
+		sl.l.With("numOutOfBounds", numOutOfBounds).Warn("Error on ingesting samples that are too old")
+	}
 	if err == nil {
 		sl.cache.forEachStale(func(lset labels.Labels) bool {
 			// Series no longer exposed, mark it stale.
diff --git a/storage/interface.go b/storage/interface.go
index c6d54c350..a1b5221a2 100644
--- a/storage/interface.go
+++ b/storage/interface.go
@@ -19,10 +19,12 @@ import (
 	"github.com/prometheus/prometheus/pkg/labels"
 )
 
+// The errors exposed.
 var (
 	ErrNotFound                    = errors.New("not found")
 	ErrOutOfOrderSample            = errors.New("out of order sample")
 	ErrDuplicateSampleForTimestamp = errors.New("duplicate sample for timestamp")
+	ErrOutOfBounds                 = errors.New("out of bounds")
 )
 
 // Storage ingests and manages samples, along with various indexes. All methods