diff --git a/promql/parser/parse.go b/promql/parser/parse.go
index 79424dd98..1b6d8a930 100644
--- a/promql/parser/parse.go
+++ b/promql/parser/parse.go
@@ -242,6 +242,11 @@ func (p *parser) addParseErr(positionRange PositionRange, err error) {
 func (p *parser) unexpected(context string, expected string) {
 	var errMsg strings.Builder
 
+	// Do not report lexer errors twice
+	if p.yyParser.lval.item.Typ == ERROR {
+		return
+	}
+
 	errMsg.WriteString("unexpected ")
 	errMsg.WriteString(p.yyParser.lval.item.desc())
 
@@ -302,10 +307,15 @@ func (p *parser) Lex(lval *yySymType) int {
 	}
 
 	switch typ {
-
 	case ERROR:
-		p.addParseErrf(lval.item.PositionRange(), "%s", lval.item.Val)
-		p.InjectItem(0)
+		pos := PositionRange{
+			Start: p.lex.start,
+			End:   Pos(len(p.lex.input)),
+		}
+		p.addParseErr(pos, errors.New(p.yyParser.lval.item.Val))
+
+		// Tells yacc that this is the end of input.
+		return 0
 	case EOF:
 		lval.item.Typ = EOF
 		p.InjectItem(0)
diff --git a/promql/parser/parse_test.go b/promql/parser/parse_test.go
index f99c4762c..2cb9467b0 100644
--- a/promql/parser/parse_test.go
+++ b/promql/parser/parse_test.go
@@ -2592,6 +2592,16 @@ func TestParseExpressions(t *testing.T) {
 		} else {
 			testutil.NotOk(t, err)
 			testutil.Assert(t, strings.Contains(err.Error(), test.errMsg), "unexpected error on input '%s', expected '%s', got '%s'", test.input, test.errMsg, err.Error())
+
+			errorList, ok := err.(ParseErrors)
+
+			testutil.Assert(t, ok, "unexpected error type")
+
+			for _, e := range errorList {
+				testutil.Assert(t, 0 <= e.PositionRange.Start, "parse error has negative position\nExpression '%s'\nError: %v", test.input, e)
+				testutil.Assert(t, e.PositionRange.Start <= e.PositionRange.End, "parse error has negative length\nExpression '%s'\nError: %v", test.input, e)
+				testutil.Assert(t, e.PositionRange.End <= Pos(len(test.input)), "parse error is not contained in input\nExpression '%s'\nError: %v", test.input, e)
+			}
 		}
 	}
 }