storage: improve index lookups

tl;dr: This is not a fundamental solution to the indexing problem
(like tindex is) but it at least avoids utilizing the intersection
problem to the greatest possible amount.

In more detail:

Imagine the following query:

    nicely:aggregating:rule{job="foo",env="prod"}

While it uses a nicely aggregating recording rule (which might have a
very low cardinality), Prometheus still intersects the low number of
fingerprints for `{__name__="nicely:aggregating:rule"}` with the many
thousands of fingerprints matching `{job="foo"}` and with the millions
of fingerprints matching `{env="prod"}`. This totally innocuous query
is dead slow if the Prometheus server has a lot of time series with
the `{env="prod"}` label. Ironically, if you make the query more
complicated, it becomes blazingly fast:

    nicely:aggregating:rule{job=~"foo",env=~"prod"}

Why so? Because Prometheus only intersects with non-Equal matchers if
there are no Equal matchers. That's good in this case because it
retrieves the few fingerprints for
`{__name__="nicely:aggregating:rule"}` and then starts right ahead to
retrieve the metric for those FPs and checking individually if they
match the other matchers.

This change is generalizing the idea of when to stop intersecting FPs
and go into "retrieve metrics and check them individually against
remaining matchers" mode:

- First, sort all matchers by "expected cardinality". Matchers
  matching the empty string are always worst (and never used for
  intersections). Equal matchers are in general consider best, but by
  using some crude heuristics, we declare some better than others
  (instance labels or anything that looks like a recording rule).

- Then go through the matchers until we hit a threshold of remaining
  FPs in the intersection. This threshold is higher if we are already
  in the non-Equal matcher area as intersection is even more expensive
  here.

- Once the threshold has been reached (or we have run out of matchers
  that do not match the empty string), start with "retrieve metrics
  and check them individually against remaining matchers".

A beefy server at SoundCloud was spending 67% of its CPU time in index
lookups (fingerprintsForLabelPairs), serving mostly a dashboard that
is exclusively built with recording rules. With this change, it spends
only 35% in fingerprintsForLabelPairs. The CPU usage dropped from 26
cores to 18 cores. The median latency for query_range dropped from 14s
to 50ms(!). As expected, higher percentile latency didn't improve that
much because the new approach is _occasionally_ running into the worst
case while the old one was _systematically_ doing so. The 99th
percentile latency is now about as high as the median before (14s)
while it was almost twice as high before (26s).

promql/parse.go

@@ -953,11 +953,11 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
 			}
 		}
 		// Set name label matching.
-		matchers = append(matchers, &metric.LabelMatcher{
-			Type:  metric.Equal,
-			Name:  model.MetricNameLabel,
-			Value: model.LabelValue(name),
-		})
+		m, err := metric.NewLabelMatcher(metric.Equal, model.MetricNameLabel, model.LabelValue(name))
+		if err != nil {
+			panic(err) // Must not happen with metric.Equal.
+		}
+		matchers = append(matchers, m)
 	}
 
 	if len(matchers) == 0 {
@@ -967,14 +967,7 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
 	// implicit selection of all metrics (e.g. by a typo).
 	notEmpty := false
 	for _, lm := range matchers {
-		// Matching changes the inner state of the regex and causes reflect.DeepEqual
-		// to return false, which break tests.
-		// Thus, we create a new label matcher for this testing.
-		lm, err := metric.NewLabelMatcher(lm.Type, lm.Name, lm.Value)
-		if err != nil {
-			p.error(err)
-		}
-		if !lm.Match("") {
+		if !lm.MatchesEmptyString() {
 			notEmpty = true
 			break
 		}

promql/parse_test.go

@@ -144,7 +144,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 		},
@@ -154,7 +154,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 		},
@@ -263,13 +263,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{Card: CardOneToOne},
@@ -281,7 +281,7 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &NumberLiteral{1},
@@ -293,7 +293,7 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS:        &NumberLiteral{1},
@@ -307,7 +307,7 @@ var testExpr = []struct {
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 		},
@@ -318,13 +318,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{Card: CardManyToMany},
@@ -336,13 +336,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{Card: CardManyToMany},
@@ -354,13 +354,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{Card: CardManyToMany},
@@ -375,13 +375,13 @@ var testExpr = []struct {
 				LHS: &VectorSelector{
 					Name: "foo",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 					},
 				},
 				RHS: &VectorSelector{
 					Name: "bar",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 					},
 				},
 				VectorMatching: &VectorMatching{Card: CardOneToOne},
@@ -391,13 +391,13 @@ var testExpr = []struct {
 				LHS: &VectorSelector{
 					Name: "bla",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
 					},
 				},
 				RHS: &VectorSelector{
 					Name: "blub",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
 					},
 				},
 				VectorMatching: &VectorMatching{Card: CardManyToMany},
@@ -416,13 +416,13 @@ var testExpr = []struct {
 					LHS: &VectorSelector{
 						Name: "foo",
 						LabelMatchers: metric.LabelMatchers{
-							{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+							mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 						},
 					},
 					RHS: &VectorSelector{
 						Name: "bar",
 						LabelMatchers: metric.LabelMatchers{
-							{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+							mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 						},
 					},
 					VectorMatching: &VectorMatching{Card: CardManyToMany},
@@ -430,7 +430,7 @@ var testExpr = []struct {
 				RHS: &VectorSelector{
 					Name: "baz",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
 					},
 				},
 				VectorMatching: &VectorMatching{Card: CardManyToMany},
@@ -438,7 +438,7 @@ var testExpr = []struct {
 			RHS: &VectorSelector{
 				Name: "qux",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "qux"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "qux"),
 				},
 			},
 			VectorMatching: &VectorMatching{Card: CardManyToMany},
@@ -451,7 +451,7 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			RHS: &BinaryExpr{
@@ -459,13 +459,13 @@ var testExpr = []struct {
 				LHS: &VectorSelector{
 					Name: "bla",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
 					},
 				},
 				RHS: &VectorSelector{
 					Name: "blub",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
 					},
 				},
 				VectorMatching: &VectorMatching{
@@ -488,13 +488,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -510,13 +510,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -532,13 +532,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -554,13 +554,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -576,13 +576,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -597,13 +597,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -618,13 +618,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "baz",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -640,13 +640,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -663,13 +663,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -685,13 +685,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -707,13 +707,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -730,13 +730,13 @@ var testExpr = []struct {
 			LHS: &VectorSelector{
 				Name: "foo",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 				},
 			},
 			RHS: &VectorSelector{
 				Name: "bar",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 				},
 			},
 			VectorMatching: &VectorMatching{
@@ -825,7 +825,7 @@ var testExpr = []struct {
 			Name:   "foo",
 			Offset: 0,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 			},
 		},
 	}, {
@@ -834,7 +834,7 @@ var testExpr = []struct {
 			Name:   "foo",
 			Offset: 5 * time.Minute,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 			},
 		},
 	}, {
@@ -843,8 +843,8 @@ var testExpr = []struct {
 			Name:   "foo:bar",
 			Offset: 0,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: "a", Value: "bc"},
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo:bar"},
+				mustLabelMatcher(metric.Equal, "a", "bc"),
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo:bar"),
 			},
 		},
 	}, {
@@ -853,8 +853,8 @@ var testExpr = []struct {
 			Name:   "foo",
 			Offset: 0,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: "NaN", Value: "bc"},
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+				mustLabelMatcher(metric.Equal, "NaN", "bc"),
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 			},
 		},
 	}, {
@@ -863,11 +863,11 @@ var testExpr = []struct {
 			Name:   "foo",
 			Offset: 0,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: "a", Value: "b"},
-				{Type: metric.NotEqual, Name: "foo", Value: "bar"},
+				mustLabelMatcher(metric.Equal, "a", "b"),
+				mustLabelMatcher(metric.NotEqual, "foo", "bar"),
 				mustLabelMatcher(metric.RegexMatch, "test", "test"),
 				mustLabelMatcher(metric.RegexNoMatch, "bar", "baz"),
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 			},
 		},
 	}, {
@@ -949,7 +949,7 @@ var testExpr = []struct {
 			Offset: 0,
 			Range:  5 * time.Second,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
 			},
 		},
 	}, {
@@ -959,7 +959,7 @@ var testExpr = []struct {
 			Offset: 0,
 			Range:  5 * time.Minute,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
 			},
 		},
 	}, {
@@ -969,7 +969,7 @@ var testExpr = []struct {
 			Offset: 5 * time.Minute,
 			Range:  5 * time.Hour,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
 			},
 		},
 	}, {
@@ -979,7 +979,7 @@ var testExpr = []struct {
 			Offset: 10 * time.Second,
 			Range:  5 * 24 * time.Hour,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
 			},
 		},
 	}, {
@@ -989,7 +989,7 @@ var testExpr = []struct {
 			Offset: 14 * 24 * time.Hour,
 			Range:  5 * 7 * 24 * time.Hour,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
 			},
 		},
 	}, {
@@ -999,8 +999,8 @@ var testExpr = []struct {
 			Offset: 3 * 24 * time.Hour,
 			Range:  5 * 365 * 24 * time.Hour,
 			LabelMatchers: metric.LabelMatchers{
-				{Type: metric.Equal, Name: "a", Value: "b"},
-				{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"},
+				mustLabelMatcher(metric.Equal, "a", "b"),
+				mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
 			},
 		},
 	}, {
@@ -1059,7 +1059,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo"},
@@ -1072,7 +1072,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo"},
@@ -1085,7 +1085,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo", "bar"},
@@ -1097,7 +1097,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo"},
@@ -1109,7 +1109,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping:         model.LabelNames{"foo"},
@@ -1122,7 +1122,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping:         model.LabelNames{"foo"},
@@ -1135,7 +1135,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo"},
@@ -1148,7 +1148,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo"},
@@ -1161,7 +1161,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo"},
@@ -1173,7 +1173,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 		},
@@ -1184,7 +1184,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{"foo"},
@@ -1196,7 +1196,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Grouping: model.LabelNames{},
@@ -1208,7 +1208,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Param: &NumberLiteral{5},
@@ -1220,7 +1220,7 @@ var testExpr = []struct {
 			Expr: &VectorSelector{
 				Name: "some_metric",
 				LabelMatchers: metric.LabelMatchers{
-					{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+					mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 				},
 			},
 			Param: &StringLiteral{"value"},
@@ -1288,8 +1288,8 @@ var testExpr = []struct {
 				&VectorSelector{
 					Name: "some_metric",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.NotEqual, Name: "foo", Value: "bar"},
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+						mustLabelMatcher(metric.NotEqual, "foo", "bar"),
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 					},
 				},
 			},
@@ -1302,7 +1302,7 @@ var testExpr = []struct {
 				&MatrixSelector{
 					Name: "some_metric",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 					},
 					Range: 5 * time.Minute,
 				},
@@ -1316,7 +1316,7 @@ var testExpr = []struct {
 				&VectorSelector{
 					Name: "some_metric",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 					},
 				},
 			},
@@ -1329,7 +1329,7 @@ var testExpr = []struct {
 				&VectorSelector{
 					Name: "some_metric",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 					},
 				},
 				&NumberLiteral{5},
@@ -1537,7 +1537,7 @@ var testStatement = []struct {
 							&MatrixSelector{
 								Name: "http_request_count",
 								LabelMatchers: metric.LabelMatchers{
-									{Type: metric.Equal, Name: model.MetricNameLabel, Value: "http_request_count"},
+									mustLabelMatcher(metric.Equal, model.MetricNameLabel, "http_request_count"),
 								},
 								Range: 5 * time.Minute,
 							},
@@ -1553,7 +1553,7 @@ var testStatement = []struct {
 					LHS: &VectorSelector{
 						Name: "dc:http_request:rate5m",
 						LabelMatchers: metric.LabelMatchers{
-							{Type: metric.Equal, Name: model.MetricNameLabel, Value: "dc:http_request:rate5m"},
+							mustLabelMatcher(metric.Equal, model.MetricNameLabel, "dc:http_request:rate5m"),
 						},
 					},
 					RHS: &NumberLiteral{10000},
@@ -1570,8 +1570,8 @@ var testStatement = []struct {
 				Expr: &VectorSelector{
 					Name: "bar",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: "label1", Value: "value1"},
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+						mustLabelMatcher(metric.Equal, "label1", "value1"),
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 					},
 				},
 				Labels: nil,
@@ -1583,7 +1583,7 @@ var testStatement = []struct {
 					LHS: &VectorSelector{
 						Name: "foo",
 						LabelMatchers: metric.LabelMatchers{
-							{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"},
+							mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
 						},
 					},
 					RHS: &NumberLiteral{10},
@@ -1604,9 +1604,9 @@ var testStatement = []struct {
 				Expr: &VectorSelector{
 					Name: "bar",
 					LabelMatchers: metric.LabelMatchers{
-						{Type: metric.Equal, Name: "a", Value: "b"},
+						mustLabelMatcher(metric.Equal, "a", "b"),
 						mustLabelMatcher(metric.RegexMatch, "x", "y"),
-						{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"},
+						mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
 					},
 				},
 				Labels: model.LabelSet{"x": "", "a": "z"},
@@ -1628,7 +1628,7 @@ var testStatement = []struct {
 					LHS: &VectorSelector{
 						Name: "some_metric",
 						LabelMatchers: metric.LabelMatchers{
-							{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"},
+							mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
 						},
 					},
 					RHS: &NumberLiteral{1},

storage/local/interface.go

@@ -60,11 +60,11 @@ type Querier interface {
 	NewPreloader() Preloader
 	// MetricsForLabelMatchers returns the metrics from storage that satisfy
 	// the given label matchers. At least one label matcher must be
-	// specified that does not match the empty string. The times from and
-	// through are hints for the storage to optimize the search. The storage
-	// MAY exclude metrics that have no samples in the specified interval
-	// from the returned map. In doubt, specify model.Earliest for from and
-	// model.Latest for through.
+	// specified that does not match the empty string, otherwise an empty
+	// map is returned. The times from and through are hints for the storage
+	// to optimize the search. The storage MAY exclude metrics that have no
+	// samples in the specified interval from the returned map. In doubt,
+	// specify model.Earliest for from and model.Latest for through.
 	MetricsForLabelMatchers(from, through model.Time, matchers ...*metric.LabelMatcher) map[model.Fingerprint]metric.Metric
 	// LastSampleForFingerprint returns the last sample that has been
 	// ingested for the provided fingerprint. If this instance of the

storage/local/storage.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -59,6 +60,19 @@ const (
 	// other words: if there are no chunks to persist, it doesn't help chunk
 	// eviction if we speed up persistence.)
 	factorMinChunksToPersist = 0.2
+
+	// Threshold for when to stop using LabelMatchers to retrieve and
+	// intersect fingerprints. The rationale here is that looking up more
+	// fingerprints has diminishing returns if we already have narrowed down
+	// the possible fingerprints significantly. It is then easier to simply
+	// lookup the metrics for all the fingerprints and directly compare them
+	// to the matchers. Since a fingerprint lookup for an Equal matcher is
+	// much less expensive, there is a lower threshold for that case.
+	// TODO(beorn7): These numbers need to be tweaked, probably a bit lower.
+	// 5x higher numbers have resulted in slightly worse performance in a
+	// real-life production scenario.
+	fpEqualMatchThreshold = 1000
+	fpOtherMatchThreshold = 10000
 )
 
 var (
@@ -445,27 +459,29 @@ func (s *MemorySeriesStorage) NewPreloader() Preloader {
 	}
 }
 
-// fingerprintsForLabelPairs returns the set of fingerprints that have the given labels.
-// This does not work with empty label values.
-func (s *MemorySeriesStorage) fingerprintsForLabelPairs(pairs ...model.LabelPair) map[model.Fingerprint]struct{} {
-	var result map[model.Fingerprint]struct{}
-	for _, pair := range pairs {
-		intersection := map[model.Fingerprint]struct{}{}
-		fps := s.persistence.fingerprintsForLabelPair(pair)
-		if len(fps) == 0 {
-			return nil
-		}
-		for _, fp := range fps {
-			if _, ok := result[fp]; ok || result == nil {
-				intersection[fp] = struct{}{}
-			}
+// fingerprintsForLabelPair returns the fingerprints with the given
+// LabelPair. If intersectWith is non-nil, the method will only return
+// fingerprints that are also contained in intersectsWith. If mergeWith is
+// non-nil, the found fingerprints are added to the given map. The returned map
+// is the same as the given one.
+func (s *MemorySeriesStorage) fingerprintsForLabelPair(
+	pair model.LabelPair,
+	mergeWith map[model.Fingerprint]struct{},
+	intersectWith map[model.Fingerprint]struct{},
+) map[model.Fingerprint]struct{} {
+	if mergeWith == nil {
+		mergeWith = map[model.Fingerprint]struct{}{}
+	}
+	for _, fp := range s.persistence.fingerprintsForLabelPair(pair) {
+		if intersectWith == nil {
+			mergeWith[fp] = struct{}{}
+			continue
 		}
-		if len(intersection) == 0 {
-			return nil
+		if _, ok := intersectWith[fp]; ok {
+			mergeWith[fp] = struct{}{}
 		}
-		result = intersection
 	}
-	return result
+	return mergeWith
 }
 
 // MetricsForLabelMatchers implements Storage.
@@ -473,68 +489,75 @@ func (s *MemorySeriesStorage) MetricsForLabelMatchers(
 	from, through model.Time,
 	matchers ...*metric.LabelMatcher,
 ) map[model.Fingerprint]metric.Metric {
+	sort.Sort(metric.LabelMatchers(matchers))
+
+	if len(matchers) == 0 || matchers[0].MatchesEmptyString() {
+		// No matchers at all or even the best matcher matches the empty string.
+		return nil
+	}
+
 	var (
-		equals  []model.LabelPair
-		filters []*metric.LabelMatcher
+		matcherIdx   int
+		remainingFPs map[model.Fingerprint]struct{}
 	)
-	for _, lm := range matchers {
-		if lm.Type == metric.Equal && lm.Value != "" {
-			equals = append(equals, model.LabelPair{
-				Name:  lm.Name,
-				Value: lm.Value,
-			})
-		} else {
-			filters = append(filters, lm)
+
+	// Equal matchers.
+	for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpEqualMatchThreshold); matcherIdx++ {
+		m := matchers[matcherIdx]
+		if m.Type != metric.Equal || m.MatchesEmptyString() {
+			break
+		}
+		remainingFPs = s.fingerprintsForLabelPair(
+			model.LabelPair{
+				Name:  m.Name,
+				Value: m.Value,
+			},
+			nil,
+			remainingFPs,
+		)
+		if len(remainingFPs) == 0 {
+			return nil
 		}
 	}
 
-	var resFPs map[model.Fingerprint]struct{}
-	if len(equals) > 0 {
-		resFPs = s.fingerprintsForLabelPairs(equals...)
-	} else {
-		// If we cannot make a preselection based on equality matchers, expanding the other matchers to labels
-		// and intersecting their fingerprints is still likely to be the best choice.
-		var remaining metric.LabelMatchers
-		for _, matcher := range filters {
-			// Equal matches are all empty values.
-			if matcher.Match("") {
-				remaining = append(remaining, matcher)
-				continue
-			}
-			intersection := map[model.Fingerprint]struct{}{}
-
-			matches := matcher.Filter(s.LabelValuesForLabelName(matcher.Name))
-			if len(matches) == 0 {
-				return nil
-			}
-			for _, v := range matches {
-				fps := s.fingerprintsForLabelPairs(model.LabelPair{
-					Name:  matcher.Name,
-					Value: v,
-				})
-				for fp := range fps {
-					if _, ok := resFPs[fp]; ok || resFPs == nil {
-						intersection[fp] = struct{}{}
-					}
-				}
-			}
-			resFPs = intersection
+	// Other matchers.
+	for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpOtherMatchThreshold); matcherIdx++ {
+		m := matchers[matcherIdx]
+		if m.MatchesEmptyString() {
+			break
+		}
+		lvs := m.Filter(s.LabelValuesForLabelName(m.Name))
+		if len(lvs) == 0 {
+			return nil
+		}
+		fps := map[model.Fingerprint]struct{}{}
+		for _, lv := range lvs {
+			s.fingerprintsForLabelPair(
+				model.LabelPair{
+					Name:  m.Name,
+					Value: lv,
+				},
+				fps,
+				remainingFPs,
+			)
+		}
+		remainingFPs = fps
+		if len(remainingFPs) == 0 {
+			return nil
 		}
-		// The intersected matchers no longer need to be compared against the actual metrics.
-		filters = remaining
 	}
 
 	result := map[model.Fingerprint]metric.Metric{}
-	for fp := range resFPs {
+	for fp := range remainingFPs {
 		s.fpLocker.Lock(fp)
 		if met, _, ok := s.metricForRange(fp, from, through); ok {
 			result[fp] = metric.Metric{Metric: met}
 		}
 		s.fpLocker.Unlock(fp)
 	}
-	for _, matcher := range filters {
+	for _, m := range matchers[matcherIdx:] {
 		for fp, met := range result {
-			if !matcher.Match(met.Metric[matcher.Name]) {
+			if !m.Match(met.Metric[m.Name]) {
 				delete(result, fp)
 			}
 		}

storage/local/storage_test.go

@@ -326,7 +326,10 @@ func TestFingerprintsForLabels(t *testing.T) {
 	}
 
 	for _, mt := range matcherTests {
-		resfps := storage.fingerprintsForLabelPairs(mt.pairs...)
+		var resfps map[model.Fingerprint]struct{}
+		for _, pair := range mt.pairs {
+			resfps = storage.fingerprintsForLabelPair(pair, nil, resfps)
+		}
 		if len(mt.expected) != len(resfps) {
 			t.Fatalf("expected %d matches for %q, found %d", len(mt.expected), mt.pairs, len(resfps))
 		}
@@ -467,7 +470,9 @@ func TestRetentionCutoff(t *testing.T) {
 	s.WaitForIndexing()
 
 	var fp model.Fingerprint
-	for f := range s.fingerprintsForLabelPairs(model.LabelPair{Name: "job", Value: "test"}) {
+	for f := range s.fingerprintsForLabelPair(model.LabelPair{
+		Name: "job", Value: "test",
+	}, nil, nil) {
 		fp = f
 		break
 	}
@@ -539,7 +544,9 @@ func TestDropMetrics(t *testing.T) {
 	s.persistence.archiveMetric(fpToBeArchived, m3, 0, insertStart.Add(time.Duration(N-1)*time.Millisecond))
 	s.fpLocker.Unlock(fpToBeArchived)
 
-	fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"})
+	fps := s.fingerprintsForLabelPair(model.LabelPair{
+		Name: model.MetricNameLabel, Value: "test",
+	}, nil, nil)
 	if len(fps) != 3 {
 		t.Errorf("unexpected number of fingerprints: %d", len(fps))
 	}
@@ -549,9 +556,9 @@ func TestDropMetrics(t *testing.T) {
 	s.DropMetricsForFingerprints(fpList[0])
 	s.WaitForIndexing()
 
-	fps2 := s.fingerprintsForLabelPairs(model.LabelPair{
+	fps2 := s.fingerprintsForLabelPair(model.LabelPair{
 		Name: model.MetricNameLabel, Value: "test",
-	})
+	}, nil, nil)
 	if len(fps2) != 2 {
 		t.Errorf("unexpected number of fingerprints: %d", len(fps2))
 	}
@@ -576,9 +583,9 @@ func TestDropMetrics(t *testing.T) {
 	s.DropMetricsForFingerprints(fpList...)
 	s.WaitForIndexing()
 
-	fps3 := s.fingerprintsForLabelPairs(model.LabelPair{
+	fps3 := s.fingerprintsForLabelPair(model.LabelPair{
 		Name: model.MetricNameLabel, Value: "test",
-	})
+	}, nil, nil)
 	if len(fps3) != 0 {
 		t.Errorf("unexpected number of fingerprints: %d", len(fps3))
 	}
@@ -658,7 +665,9 @@ func TestQuarantineMetric(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"})
+	fps := s.fingerprintsForLabelPair(model.LabelPair{
+		Name: model.MetricNameLabel, Value: "test",
+	}, nil, nil)
 	if len(fps) != 3 {
 		t.Errorf("unexpected number of fingerprints: %d", len(fps))
 	}
@@ -670,9 +679,9 @@ func TestQuarantineMetric(t *testing.T) {
 	time.Sleep(time.Second) // Give time to quarantine. TODO(beorn7): Find a better way to wait.
 	s.WaitForIndexing()
 
-	fps2 := s.fingerprintsForLabelPairs(model.LabelPair{
+	fps2 := s.fingerprintsForLabelPair(model.LabelPair{
 		Name: model.MetricNameLabel, Value: "test",
-	})
+	}, nil, nil)
 	if len(fps2) != 2 {
 		t.Errorf("unexpected number of fingerprints: %d", len(fps2))
 	}

storage/metric/matcher.go

@@ -16,6 +16,7 @@ package metric
 import (
 	"fmt"
 	"regexp"
+	"strings"
 
 	"github.com/prometheus/common/model"
 )
@@ -44,15 +45,23 @@ func (m MatchType) String() string {
 	panic("unknown match type")
 }
 
-// LabelMatchers is a slice of LabelMatcher objects.
+// LabelMatchers is a slice of LabelMatcher objects. By implementing the
+// sort.Interface, it is sortable by cardinality score, i.e. after sorting, the
+// LabelMatcher that is expected to yield the fewest matches is first in the
+// slice, and LabelMatchers that match the empty string are last.
 type LabelMatchers []*LabelMatcher
 
-// LabelMatcher models the matching of a label.
+func (lms LabelMatchers) Len() int           { return len(lms) }
+func (lms LabelMatchers) Swap(i, j int)      { lms[i], lms[j] = lms[j], lms[i] }
+func (lms LabelMatchers) Less(i, j int) bool { return lms[i].score < lms[j].score }
+
+// LabelMatcher models the matching of a label. Create with NewLabelMatcher.
 type LabelMatcher struct {
 	Type  MatchType
 	Name  model.LabelName
 	Value model.LabelValue
 	re    *regexp.Regexp
+	score float64 // Cardinality score, between 0 and 1, 0 is lowest cardinality.
 }
 
 // NewLabelMatcher returns a LabelMatcher object ready to use.
@@ -69,9 +78,96 @@ func NewLabelMatcher(matchType MatchType, name model.LabelName, value model.Labe
 		}
 		m.re = re
 	}
+	m.calculateScore()
 	return m, nil
 }
 
+// calculateScore is a helper method only called in the constructor. It
+// calculates the cardinality score upfront, so that sorting by it is faster and
+// doesn't change internal state of the matcher.
+//
+// The score is based on a pretty bad but still quite helpful heuristics for
+// now. Note that this is an interim solution until the work in progress to
+// properly intersect matchers is complete. We intend to not invest any further
+// effort into tweaking the score calculation, as this could easily devolve into
+// a rabbit hole.
+//
+// The heuristics works along the following lines:
+//
+// - A matcher that is known to match nothing would have a score of 0. (This
+//   case doesn't happen in the scope of this method.)
+//
+// - A matcher that matches the empty string has a score of 1.
+//
+// - Equal matchers have a score <= 0.5. The order in score for other matchers
+//   are RegexMatch, RegexNoMatch, NotEqual.
+//
+// - There are a number of score adjustments for known "magic" parts, like
+//   instance labels, metric names containing a colon (which are probably
+//   recording rules) and such.
+//
+// - On top, there is a tiny adjustment for the length of the matcher, following
+//   the blunt expectation that a long label name and/or value is more specific
+//   and will therefore have a lower cardinality.
+//
+// To reiterate on the above: PLEASE RESIST THE TEMPTATION TO TWEAK THIS
+// METHOD. IT IS "MAGIC" ENOUGH ALREADY AND WILL GO AWAY WITH THE UPCOMING MORE
+// POWERFUL INDEXING.
+func (m *LabelMatcher) calculateScore() {
+	if m.Match("") {
+		m.score = 1
+		return
+	}
+	// lengthCorrection is between 0 (for length 0) and 0.1 (for length +Inf).
+	lengthCorrection := 0.1 * (1 - 1/float64(len(m.Name)+len(m.Value)+1))
+	switch m.Type {
+	case Equal:
+		m.score = 0.3 - lengthCorrection
+	case RegexMatch:
+		m.score = 0.6 - lengthCorrection
+	case RegexNoMatch:
+		m.score = 0.8 + lengthCorrection
+	case NotEqual:
+		m.score = 0.9 + lengthCorrection
+	}
+	if m.Type != Equal {
+		// Don't bother anymore in this case.
+		return
+	}
+	switch m.Name {
+	case model.InstanceLabel:
+		// Matches only metrics from a single instance, which clearly
+		// limits the damage.
+		m.score -= 0.2
+	case model.JobLabel:
+		// The usual case is a relatively low number of jobs with many
+		// metrics each.
+		m.score += 0.1
+	case model.BucketLabel, model.QuantileLabel:
+		// Magic labels for buckets and quantiles will match copiously.
+		m.score += 0.2
+	case model.MetricNameLabel:
+		if strings.Contains(string(m.Value), ":") {
+			// Probably a recording rule with limited cardinality.
+			m.score -= 0.1
+			return
+		}
+		if m.Value == "up" || m.Value == "scrape_duration_seconds" {
+			// Synthetic metrics which are contained in every scrape
+			// exactly once.  There might be less frequent metric
+			// names, but the worst case is limited here, so give it
+			// a bump.
+			m.score -= 0.05
+			return
+		}
+	}
+}
+
+// MatchesEmptyString returns true if the LabelMatcher matches the empty string.
+func (m *LabelMatcher) MatchesEmptyString() bool {
+	return m.score >= 1
+}
+
 func (m *LabelMatcher) String() string {
 	return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value)
 }

storage/metric/matcher_test.go

@@ -14,7 +14,12 @@
 package metric
 
 import (
+	"math/rand"
+	"reflect"
+	"sort"
 	"testing"
+
+	"github.com/prometheus/common/model"
 )
 
 func TestAnchoredMatcher(t *testing.T) {
@@ -30,3 +35,39 @@ func TestAnchoredMatcher(t *testing.T) {
 		t.Errorf("Unexpected match for %q", "fooo")
 	}
 }
+
+func TestLabelMatchersSort(t *testing.T) {
+	// Line up Matchers in expected order:
+	want := LabelMatchers{
+		mustNewLabelMatcher(Equal, model.InstanceLabel, "not empty"),
+		mustNewLabelMatcher(Equal, model.MetricNameLabel, "a:recording:rule"),
+		mustNewLabelMatcher(Equal, model.MetricNameLabel, "up"),
+		mustNewLabelMatcher(Equal, "nothing_special but much longer", "not empty"),
+		mustNewLabelMatcher(Equal, "nothing_special", "not empty but longer"),
+		mustNewLabelMatcher(Equal, "nothing_special", "not empty"),
+		mustNewLabelMatcher(Equal, model.JobLabel, "not empty"),
+		mustNewLabelMatcher(Equal, model.BucketLabel, "not empty"),
+		mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string and is longer"),
+		mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string"),
+		mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string)?"),
+		mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string with a longer expression)?"),
+		mustNewLabelMatcher(NotEqual, "irrelevant", ""),
+		mustNewLabelMatcher(Equal, "irrelevant", ""),
+	}
+	got := make(LabelMatchers, len(want))
+	for i, j := range rand.Perm(len(want)) {
+		got[i] = want[j]
+	}
+	sort.Sort(got)
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("unexpected sorting of matchers, got %v, want %v", got, want)
+	}
+}
+
+func mustNewLabelMatcher(mt MatchType, name model.LabelName, val model.LabelValue) *LabelMatcher {
+	m, err := NewLabelMatcher(mt, name, val)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}