github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

storage: improve index lookups

Browse Source 
tl;dr: This is not a fundamental solution to the indexing problem
(like tindex is) but it at least avoids utilizing the intersection
problem to the greatest possible amount.

In more detail:

Imagine the following query:

    nicely:aggregating:rule{job="foo",env="prod"}

While it uses a nicely aggregating recording rule (which might have a
very low cardinality), Prometheus still intersects the low number of
fingerprints for `{__name__="nicely:aggregating:rule"}` with the many
thousands of fingerprints matching `{job="foo"}` and with the millions
of fingerprints matching `{env="prod"}`. This totally innocuous query
is dead slow if the Prometheus server has a lot of time series with
the `{env="prod"}` label. Ironically, if you make the query more
complicated, it becomes blazingly fast:

    nicely:aggregating:rule{job=~"foo",env=~"prod"}

Why so? Because Prometheus only intersects with non-Equal matchers if
there are no Equal matchers. That's good in this case because it
retrieves the few fingerprints for
`{__name__="nicely:aggregating:rule"}` and then starts right ahead to
retrieve the metric for those FPs and checking individually if they
match the other matchers.

This change is generalizing the idea of when to stop intersecting FPs
and go into "retrieve metrics and check them individually against
remaining matchers" mode:

- First, sort all matchers by "expected cardinality". Matchers
  matching the empty string are always worst (and never used for
  intersections). Equal matchers are in general consider best, but by
  using some crude heuristics, we declare some better than others
  (instance labels or anything that looks like a recording rule).

- Then go through the matchers until we hit a threshold of remaining
  FPs in the intersection. This threshold is higher if we are already
  in the non-Equal matcher area as intersection is even more expensive
  here.

- Once the threshold has been reached (or we have run out of matchers
  that do not match the empty string), start with "retrieve metrics
  and check them individually against remaining matchers".

A beefy server at SoundCloud was spending 67% of its CPU time in index
lookups (fingerprintsForLabelPairs), serving mostly a dashboard that
is exclusively built with recording rules. With this change, it spends
only 35% in fingerprintsForLabelPairs. The CPU usage dropped from 26
cores to 18 cores. The median latency for query_range dropped from 14s
to 50ms(!). As expected, higher percentile latency didn't improve that
much because the new approach is _occasionally_ running into the worst
case while the old one was _systematically_ doing so. The 99th
percentile latency is now about as high as the median before (14s)
while it was almost twice as high before (26s).
pull/1774/head
beorn7 9 years ago
parent 40f8da699e
commit fc6737b7fb
7 changed files with 347 additions and 185 deletions
Show Stats Download Patch File Download Diff File

19
promql/parse.go
Unescape View File

@ -953,11 +953,11 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
} }
} }
// Set name label matching. // Set name label matching.
matchers = append(matchers, &metric.LabelMatcher{ m, err := metric.NewLabelMatcher(metric.Equal, model.MetricNameLabel, model.LabelValue(name))
Type: metric.Equal, if err != nil {
Name: model.MetricNameLabel, panic(err) // Must not happen with metric.Equal.
Value: model.LabelValue(name), }
}) matchers = append(matchers, m)
} }
if len(matchers) == 0 { if len(matchers) == 0 {
@ -967,14 +967,7 @@ func (p *parser) vectorSelector(name string) *VectorSelector {
// implicit selection of all metrics (e.g. by a typo). // implicit selection of all metrics (e.g. by a typo).
notEmpty := false notEmpty := false
for _, lm := range matchers { for _, lm := range matchers {
// Matching changes the inner state of the regex and causes reflect.DeepEqual if !lm.MatchesEmptyString() {
// to return false, which break tests.
// Thus, we create a new label matcher for this testing.
lm, err := metric.NewLabelMatcher(lm.Type, lm.Name, lm.Value)
if err != nil {
p.error(err)
}
if !lm.Match("") {
notEmpty = true notEmpty = true
break break
} }

182
promql/parse_test.go
Unescape View File

@ -144,7 +144,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -154,7 +154,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -263,13 +263,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardOneToOne}, VectorMatching: &VectorMatching{Card: CardOneToOne},
@ -281,7 +281,7 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &NumberLiteral{1}, RHS: &NumberLiteral{1},
@ -293,7 +293,7 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &NumberLiteral{1}, RHS: &NumberLiteral{1},
@ -307,7 +307,7 @@ var testExpr = []struct {
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
}, },
@ -318,13 +318,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -336,13 +336,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -354,13 +354,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -375,13 +375,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardOneToOne}, VectorMatching: &VectorMatching{Card: CardOneToOne},
@ -391,13 +391,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "bla", Name: "bla",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "blub", Name: "blub",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -416,13 +416,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -430,7 +430,7 @@ var testExpr = []struct {
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "baz", Name: "baz",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -438,7 +438,7 @@ var testExpr = []struct {
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "qux", Name: "qux",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "qux"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "qux"),
}, },
}, },
VectorMatching: &VectorMatching{Card: CardManyToMany}, VectorMatching: &VectorMatching{Card: CardManyToMany},
@ -451,7 +451,7 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
RHS: &BinaryExpr{ RHS: &BinaryExpr{
@ -459,13 +459,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "bla", Name: "bla",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bla"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bla"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "blub", Name: "blub",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "blub"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "blub"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -488,13 +488,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -510,13 +510,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -532,13 +532,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -554,13 +554,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -576,13 +576,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -597,13 +597,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -618,13 +618,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "baz", Name: "baz",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "baz"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "baz"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -640,13 +640,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -663,13 +663,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -685,13 +685,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -707,13 +707,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -730,13 +730,13 @@ var testExpr = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &VectorSelector{ RHS: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
VectorMatching: &VectorMatching{ VectorMatching: &VectorMatching{
@ -825,7 +825,7 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -834,7 +834,7 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 5 * time.Minute, Offset: 5 * time.Minute,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -843,8 +843,8 @@ var testExpr = []struct {
Name: "foo:bar", Name: "foo:bar",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "bc"}, mustLabelMatcher(metric.Equal, "a", "bc"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo:bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo:bar"),
}, },
}, },
}, { }, {
@ -853,8 +853,8 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "NaN", Value: "bc"}, mustLabelMatcher(metric.Equal, "NaN", "bc"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -863,11 +863,11 @@ var testExpr = []struct {
Name: "foo", Name: "foo",
Offset: 0, Offset: 0,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"}, mustLabelMatcher(metric.Equal, "a", "b"),
{Type: metric.NotEqual, Name: "foo", Value: "bar"}, mustLabelMatcher(metric.NotEqual, "foo", "bar"),
mustLabelMatcher(metric.RegexMatch, "test", "test"), mustLabelMatcher(metric.RegexMatch, "test", "test"),
mustLabelMatcher(metric.RegexNoMatch, "bar", "baz"), mustLabelMatcher(metric.RegexNoMatch, "bar", "baz"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
}, { }, {
@ -949,7 +949,7 @@ var testExpr = []struct {
Offset: 0, Offset: 0,
Range: 5 * time.Second, Range: 5 * time.Second,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -959,7 +959,7 @@ var testExpr = []struct {
Offset: 0, Offset: 0,
Range: 5 * time.Minute, Range: 5 * time.Minute,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -969,7 +969,7 @@ var testExpr = []struct {
Offset: 5 * time.Minute, Offset: 5 * time.Minute,
Range: 5 * time.Hour, Range: 5 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -979,7 +979,7 @@ var testExpr = []struct {
Offset: 10 * time.Second, Offset: 10 * time.Second,
Range: 5 * 24 * time.Hour, Range: 5 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -989,7 +989,7 @@ var testExpr = []struct {
Offset: 14 * 24 * time.Hour, Offset: 14 * 24 * time.Hour,
Range: 5 * 7 * 24 * time.Hour, Range: 5 * 7 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -999,8 +999,8 @@ var testExpr = []struct {
Offset: 3 * 24 * time.Hour, Offset: 3 * 24 * time.Hour,
Range: 5 * 365 * 24 * time.Hour, Range: 5 * 365 * 24 * time.Hour,
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"}, mustLabelMatcher(metric.Equal, "a", "b"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "test"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "test"),
}, },
}, },
}, { }, {
@ -1059,7 +1059,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1072,7 +1072,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1085,7 +1085,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo", "bar"}, Grouping: model.LabelNames{"foo", "bar"},
@ -1097,7 +1097,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1109,7 +1109,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1122,7 +1122,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1135,7 +1135,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1148,7 +1148,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1161,7 +1161,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1173,7 +1173,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -1184,7 +1184,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{"foo"}, Grouping: model.LabelNames{"foo"},
@ -1196,7 +1196,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Grouping: model.LabelNames{}, Grouping: model.LabelNames{},
@ -1208,7 +1208,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Param: &NumberLiteral{5}, Param: &NumberLiteral{5},
@ -1220,7 +1220,7 @@ var testExpr = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
Param: &StringLiteral{"value"}, Param: &StringLiteral{"value"},
@ -1288,8 +1288,8 @@ var testExpr = []struct {
&VectorSelector{ &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.NotEqual, Name: "foo", Value: "bar"}, mustLabelMatcher(metric.NotEqual, "foo", "bar"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -1302,7 +1302,7 @@ var testExpr = []struct {
&MatrixSelector{ &MatrixSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
Range: 5 * time.Minute, Range: 5 * time.Minute,
}, },
@ -1316,7 +1316,7 @@ var testExpr = []struct {
&VectorSelector{ &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
}, },
@ -1329,7 +1329,7 @@ var testExpr = []struct {
&VectorSelector{ &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
&NumberLiteral{5}, &NumberLiteral{5},
@ -1537,7 +1537,7 @@ var testStatement = []struct {
&MatrixSelector{ &MatrixSelector{
Name: "http_request_count", Name: "http_request_count",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "http_request_count"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "http_request_count"),
}, },
Range: 5 * time.Minute, Range: 5 * time.Minute,
}, },
@ -1553,7 +1553,7 @@ var testStatement = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "dc:http_request:rate5m", Name: "dc:http_request:rate5m",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "dc:http_request:rate5m"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "dc:http_request:rate5m"),
}, },
}, },
RHS: &NumberLiteral{10000}, RHS: &NumberLiteral{10000},
@ -1570,8 +1570,8 @@ var testStatement = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "label1", Value: "value1"}, mustLabelMatcher(metric.Equal, "label1", "value1"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
Labels: nil, Labels: nil,
@ -1583,7 +1583,7 @@ var testStatement = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "foo", Name: "foo",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "foo"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "foo"),
}, },
}, },
RHS: &NumberLiteral{10}, RHS: &NumberLiteral{10},
@ -1604,9 +1604,9 @@ var testStatement = []struct {
Expr: &VectorSelector{ Expr: &VectorSelector{
Name: "bar", Name: "bar",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: "a", Value: "b"}, mustLabelMatcher(metric.Equal, "a", "b"),
mustLabelMatcher(metric.RegexMatch, "x", "y"), mustLabelMatcher(metric.RegexMatch, "x", "y"),
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "bar"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "bar"),
}, },
}, },
Labels: model.LabelSet{"x": "", "a": "z"}, Labels: model.LabelSet{"x": "", "a": "z"},
@ -1628,7 +1628,7 @@ var testStatement = []struct {
LHS: &VectorSelector{ LHS: &VectorSelector{
Name: "some_metric", Name: "some_metric",
LabelMatchers: metric.LabelMatchers{ LabelMatchers: metric.LabelMatchers{
{Type: metric.Equal, Name: model.MetricNameLabel, Value: "some_metric"}, mustLabelMatcher(metric.Equal, model.MetricNameLabel, "some_metric"),
}, },
}, },
RHS: &NumberLiteral{1}, RHS: &NumberLiteral{1},

10
storage/local/interface.go
Unescape View File

@ -60,11 +60,11 @@ type Querier interface {
NewPreloader() Preloader NewPreloader() Preloader
// MetricsForLabelMatchers returns the metrics from storage that satisfy // MetricsForLabelMatchers returns the metrics from storage that satisfy
// the given label matchers. At least one label matcher must be // the given label matchers. At least one label matcher must be
// specified that does not match the empty string. The times from and // specified that does not match the empty string, otherwise an empty
// through are hints for the storage to optimize the search. The storage // map is returned. The times from and through are hints for the storage
// MAY exclude metrics that have no samples in the specified interval // to optimize the search. The storage MAY exclude metrics that have no
// from the returned map. In doubt, specify model.Earliest for from and // samples in the specified interval from the returned map. In doubt,
// model.Latest for through. // specify model.Earliest for from and model.Latest for through.
MetricsForLabelMatchers(from, through model.Time, matchers ...*metric.LabelMatcher) map[model.Fingerprint]metric.Metric MetricsForLabelMatchers(from, through model.Time, matchers ...*metric.LabelMatcher) map[model.Fingerprint]metric.Metric
// LastSampleForFingerprint returns the last sample that has been // LastSampleForFingerprint returns the last sample that has been
// ingested for the provided fingerprint. If this instance of the // ingested for the provided fingerprint. If this instance of the

151
storage/local/storage.go
Unescape View File

@ -19,6 +19,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"math" "math"
"sort"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
@ -59,6 +60,19 @@ const (
// other words: if there are no chunks to persist, it doesn't help chunk // other words: if there are no chunks to persist, it doesn't help chunk
// eviction if we speed up persistence.) // eviction if we speed up persistence.)
factorMinChunksToPersist = 0.2 factorMinChunksToPersist = 0.2
// Threshold for when to stop using LabelMatchers to retrieve and
// intersect fingerprints. The rationale here is that looking up more
// fingerprints has diminishing returns if we already have narrowed down
// the possible fingerprints significantly. It is then easier to simply
// lookup the metrics for all the fingerprints and directly compare them
// to the matchers. Since a fingerprint lookup for an Equal matcher is
// much less expensive, there is a lower threshold for that case.
// TODO(beorn7): These numbers need to be tweaked, probably a bit lower.
// 5x higher numbers have resulted in slightly worse performance in a
// real-life production scenario.
fpEqualMatchThreshold = 1000
fpOtherMatchThreshold = 10000
) )
var ( var (
@ -445,27 +459,29 @@ func (s *MemorySeriesStorage) NewPreloader() Preloader {
} }
} }
// fingerprintsForLabelPairs returns the set of fingerprints that have the given labels. // fingerprintsForLabelPair returns the fingerprints with the given
// This does not work with empty label values. // LabelPair. If intersectWith is non-nil, the method will only return
func (s *MemorySeriesStorage) fingerprintsForLabelPairs(pairs ...model.LabelPair) map[model.Fingerprint]struct{} { // fingerprints that are also contained in intersectsWith. If mergeWith is
var result map[model.Fingerprint]struct{} // non-nil, the found fingerprints are added to the given map. The returned map
for _, pair := range pairs { // is the same as the given one.
intersection := map[model.Fingerprint]struct{}{} func (s *MemorySeriesStorage) fingerprintsForLabelPair(
fps := s.persistence.fingerprintsForLabelPair(pair) pair model.LabelPair,
if len(fps) == 0 { mergeWith map[model.Fingerprint]struct{},
return nil intersectWith map[model.Fingerprint]struct{},
} ) map[model.Fingerprint]struct{} {
for _, fp := range fps { if mergeWith == nil {
if _, ok := result[fp]; ok || result == nil { mergeWith = map[model.Fingerprint]struct{}{}
intersection[fp] = struct{}{} }
} for _, fp := range s.persistence.fingerprintsForLabelPair(pair) {
if intersectWith == nil {
mergeWith[fp] = struct{}{}
continue
} }
if len(intersection) == 0 { if _, ok := intersectWith[fp]; ok {
return nil mergeWith[fp] = struct{}{}
} }
result = intersection
} }
return result return mergeWith
} }
// MetricsForLabelMatchers implements Storage. // MetricsForLabelMatchers implements Storage.
@ -473,68 +489,75 @@ func (s *MemorySeriesStorage) MetricsForLabelMatchers(
from, through model.Time, from, through model.Time,
matchers ...*metric.LabelMatcher, matchers ...*metric.LabelMatcher,
) map[model.Fingerprint]metric.Metric { ) map[model.Fingerprint]metric.Metric {
sort.Sort(metric.LabelMatchers(matchers))
if len(matchers) == 0 || matchers[0].MatchesEmptyString() {
// No matchers at all or even the best matcher matches the empty string.
return nil
}
var ( var (
equals []model.LabelPair matcherIdx int
filters []*metric.LabelMatcher remainingFPs map[model.Fingerprint]struct{}
) )
for _, lm := range matchers {
if lm.Type == metric.Equal && lm.Value != "" { // Equal matchers.
equals = append(equals, model.LabelPair{ for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpEqualMatchThreshold); matcherIdx++ {
Name: lm.Name, m := matchers[matcherIdx]
Value: lm.Value, if m.Type != metric.Equal || m.MatchesEmptyString() {
}) break
} else { }
filters = append(filters, lm) remainingFPs = s.fingerprintsForLabelPair(
model.LabelPair{
Name: m.Name,
Value: m.Value,
},
nil,
remainingFPs,
)
if len(remainingFPs) == 0 {
return nil
} }
} }
var resFPs map[model.Fingerprint]struct{} // Other matchers.
if len(equals) > 0 { for ; matcherIdx < len(matchers) && (remainingFPs == nil || len(remainingFPs) > fpOtherMatchThreshold); matcherIdx++ {
resFPs = s.fingerprintsForLabelPairs(equals...) m := matchers[matcherIdx]
} else { if m.MatchesEmptyString() {
// If we cannot make a preselection based on equality matchers, expanding the other matchers to labels break
// and intersecting their fingerprints is still likely to be the best choice. }
var remaining metric.LabelMatchers lvs := m.Filter(s.LabelValuesForLabelName(m.Name))
for _, matcher := range filters { if len(lvs) == 0 {
// Equal matches are all empty values. return nil
if matcher.Match("") { }
remaining = append(remaining, matcher) fps := map[model.Fingerprint]struct{}{}
continue for _, lv := range lvs {
} s.fingerprintsForLabelPair(
intersection := map[model.Fingerprint]struct{}{} model.LabelPair{
Name: m.Name,
matches := matcher.Filter(s.LabelValuesForLabelName(matcher.Name)) Value: lv,
if len(matches) == 0 { },
return nil fps,
} remainingFPs,
for _, v := range matches { )
fps := s.fingerprintsForLabelPairs(model.LabelPair{ }
Name: matcher.Name, remainingFPs = fps
Value: v, if len(remainingFPs) == 0 {
}) return nil
for fp := range fps {
if _, ok := resFPs[fp]; ok || resFPs == nil {
intersection[fp] = struct{}{}
}
}
}
resFPs = intersection
} }
// The intersected matchers no longer need to be compared against the actual metrics.
filters = remaining
} }
result := map[model.Fingerprint]metric.Metric{} result := map[model.Fingerprint]metric.Metric{}
for fp := range resFPs { for fp := range remainingFPs {
s.fpLocker.Lock(fp) s.fpLocker.Lock(fp)
if met, _, ok := s.metricForRange(fp, from, through); ok { if met, _, ok := s.metricForRange(fp, from, through); ok {
result[fp] = metric.Metric{Metric: met} result[fp] = metric.Metric{Metric: met}
} }
s.fpLocker.Unlock(fp) s.fpLocker.Unlock(fp)
} }
for _, matcher := range filters { for _, m := range matchers[matcherIdx:] {
for fp, met := range result { for fp, met := range result {
if !matcher.Match(met.Metric[matcher.Name]) { if !m.Match(met.Metric[m.Name]) {
delete(result, fp) delete(result, fp)
} }
} }

29
storage/local/storage_test.go
Unescape View File

@ -326,7 +326,10 @@ func TestFingerprintsForLabels(t *testing.T) {
} }
for _, mt := range matcherTests { for _, mt := range matcherTests {
resfps := storage.fingerprintsForLabelPairs(mt.pairs...) var resfps map[model.Fingerprint]struct{}
for _, pair := range mt.pairs {
resfps = storage.fingerprintsForLabelPair(pair, nil, resfps)
}
if len(mt.expected) != len(resfps) { if len(mt.expected) != len(resfps) {
t.Fatalf("expected %d matches for %q, found %d", len(mt.expected), mt.pairs, len(resfps)) t.Fatalf("expected %d matches for %q, found %d", len(mt.expected), mt.pairs, len(resfps))
} }
@ -467,7 +470,9 @@ func TestRetentionCutoff(t *testing.T) {
s.WaitForIndexing() s.WaitForIndexing()
var fp model.Fingerprint var fp model.Fingerprint
for f := range s.fingerprintsForLabelPairs(model.LabelPair{Name: "job", Value: "test"}) { for f := range s.fingerprintsForLabelPair(model.LabelPair{
Name: "job", Value: "test",
}, nil, nil) {
fp = f fp = f
break break
} }
@ -539,7 +544,9 @@ func TestDropMetrics(t *testing.T) {
s.persistence.archiveMetric(fpToBeArchived, m3, 0, insertStart.Add(time.Duration(N-1)*time.Millisecond)) s.persistence.archiveMetric(fpToBeArchived, m3, 0, insertStart.Add(time.Duration(N-1)*time.Millisecond))
s.fpLocker.Unlock(fpToBeArchived) s.fpLocker.Unlock(fpToBeArchived)
fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"}) fps := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
}, nil, nil)
if len(fps) != 3 { if len(fps) != 3 {
t.Errorf("unexpected number of fingerprints: %d", len(fps)) t.Errorf("unexpected number of fingerprints: %d", len(fps))
} }
@ -549,9 +556,9 @@ func TestDropMetrics(t *testing.T) {
s.DropMetricsForFingerprints(fpList[0]) s.DropMetricsForFingerprints(fpList[0])
s.WaitForIndexing() s.WaitForIndexing()
fps2 := s.fingerprintsForLabelPairs(model.LabelPair{ fps2 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test", Name: model.MetricNameLabel, Value: "test",
}) }, nil, nil)
if len(fps2) != 2 { if len(fps2) != 2 {
t.Errorf("unexpected number of fingerprints: %d", len(fps2)) t.Errorf("unexpected number of fingerprints: %d", len(fps2))
} }
@ -576,9 +583,9 @@ func TestDropMetrics(t *testing.T) {
s.DropMetricsForFingerprints(fpList...) s.DropMetricsForFingerprints(fpList...)
s.WaitForIndexing() s.WaitForIndexing()
fps3 := s.fingerprintsForLabelPairs(model.LabelPair{ fps3 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test", Name: model.MetricNameLabel, Value: "test",
}) }, nil, nil)
if len(fps3) != 0 { if len(fps3) != 0 {
t.Errorf("unexpected number of fingerprints: %d", len(fps3)) t.Errorf("unexpected number of fingerprints: %d", len(fps3))
} }
@ -658,7 +665,9 @@ func TestQuarantineMetric(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
fps := s.fingerprintsForLabelPairs(model.LabelPair{Name: model.MetricNameLabel, Value: "test"}) fps := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test",
}, nil, nil)
if len(fps) != 3 { if len(fps) != 3 {
t.Errorf("unexpected number of fingerprints: %d", len(fps)) t.Errorf("unexpected number of fingerprints: %d", len(fps))
} }
@ -670,9 +679,9 @@ func TestQuarantineMetric(t *testing.T) {
time.Sleep(time.Second) // Give time to quarantine. TODO(beorn7): Find a better way to wait. time.Sleep(time.Second) // Give time to quarantine. TODO(beorn7): Find a better way to wait.
s.WaitForIndexing() s.WaitForIndexing()
fps2 := s.fingerprintsForLabelPairs(model.LabelPair{ fps2 := s.fingerprintsForLabelPair(model.LabelPair{
Name: model.MetricNameLabel, Value: "test", Name: model.MetricNameLabel, Value: "test",
}) }, nil, nil)
if len(fps2) != 2 { if len(fps2) != 2 {
t.Errorf("unexpected number of fingerprints: %d", len(fps2)) t.Errorf("unexpected number of fingerprints: %d", len(fps2))
} }

100
storage/metric/matcher.go
Unescape View File

@ -16,6 +16,7 @@ package metric
import ( import (
"fmt" "fmt"
"regexp" "regexp"
"strings"
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
) )
@ -44,15 +45,23 @@ func (m MatchType) String() string {
panic("unknown match type") panic("unknown match type")
} }
// LabelMatchers is a slice of LabelMatcher objects. // LabelMatchers is a slice of LabelMatcher objects. By implementing the
// sort.Interface, it is sortable by cardinality score, i.e. after sorting, the
// LabelMatcher that is expected to yield the fewest matches is first in the
// slice, and LabelMatchers that match the empty string are last.
type LabelMatchers []*LabelMatcher type LabelMatchers []*LabelMatcher
// LabelMatcher models the matching of a label. func (lms LabelMatchers) Len() int { return len(lms) }
func (lms LabelMatchers) Swap(i, j int) { lms[i], lms[j] = lms[j], lms[i] }
func (lms LabelMatchers) Less(i, j int) bool { return lms[i].score < lms[j].score }
// LabelMatcher models the matching of a label. Create with NewLabelMatcher.
type LabelMatcher struct { type LabelMatcher struct {
Type MatchType Type MatchType
Name model.LabelName Name model.LabelName
Value model.LabelValue Value model.LabelValue
re *regexp.Regexp re *regexp.Regexp
score float64 // Cardinality score, between 0 and 1, 0 is lowest cardinality.
} }
// NewLabelMatcher returns a LabelMatcher object ready to use. // NewLabelMatcher returns a LabelMatcher object ready to use.
@ -69,9 +78,96 @@ func NewLabelMatcher(matchType MatchType, name model.LabelName, value model.Labe
} }
m.re = re m.re = re
} }
m.calculateScore()
return m, nil return m, nil
} }
// calculateScore is a helper method only called in the constructor. It
// calculates the cardinality score upfront, so that sorting by it is faster and
// doesn't change internal state of the matcher.
//
// The score is based on a pretty bad but still quite helpful heuristics for
// now. Note that this is an interim solution until the work in progress to
// properly intersect matchers is complete. We intend to not invest any further
// effort into tweaking the score calculation, as this could easily devolve into
// a rabbit hole.
//
// The heuristics works along the following lines:
//
// - A matcher that is known to match nothing would have a score of 0. (This
// case doesn't happen in the scope of this method.)
//
// - A matcher that matches the empty string has a score of 1.
//
// - Equal matchers have a score <= 0.5. The order in score for other matchers
// are RegexMatch, RegexNoMatch, NotEqual.
//
// - There are a number of score adjustments for known "magic" parts, like
// instance labels, metric names containing a colon (which are probably
// recording rules) and such.
//
// - On top, there is a tiny adjustment for the length of the matcher, following
// the blunt expectation that a long label name and/or value is more specific
// and will therefore have a lower cardinality.
//
// To reiterate on the above: PLEASE RESIST THE TEMPTATION TO TWEAK THIS
// METHOD. IT IS "MAGIC" ENOUGH ALREADY AND WILL GO AWAY WITH THE UPCOMING MORE
// POWERFUL INDEXING.
func (m *LabelMatcher) calculateScore() {
if m.Match("") {
m.score = 1
return
}
// lengthCorrection is between 0 (for length 0) and 0.1 (for length +Inf).
lengthCorrection := 0.1 * (1 - 1/float64(len(m.Name)+len(m.Value)+1))
switch m.Type {
case Equal:
m.score = 0.3 - lengthCorrection
case RegexMatch:
m.score = 0.6 - lengthCorrection
case RegexNoMatch:
m.score = 0.8 + lengthCorrection
case NotEqual:
m.score = 0.9 + lengthCorrection
}
if m.Type != Equal {
// Don't bother anymore in this case.
return
}
switch m.Name {
case model.InstanceLabel:
// Matches only metrics from a single instance, which clearly
// limits the damage.
m.score -= 0.2
case model.JobLabel:
// The usual case is a relatively low number of jobs with many
// metrics each.
m.score += 0.1
case model.BucketLabel, model.QuantileLabel:
// Magic labels for buckets and quantiles will match copiously.
m.score += 0.2
case model.MetricNameLabel:
if strings.Contains(string(m.Value), ":") {
// Probably a recording rule with limited cardinality.
m.score -= 0.1
return
}
if m.Value == "up" || m.Value == "scrape_duration_seconds" {
// Synthetic metrics which are contained in every scrape
// exactly once. There might be less frequent metric
// names, but the worst case is limited here, so give it
// a bump.
m.score -= 0.05
return
}
}
}
// MatchesEmptyString returns true if the LabelMatcher matches the empty string.
func (m *LabelMatcher) MatchesEmptyString() bool {
return m.score >= 1
}
func (m *LabelMatcher) String() string { func (m *LabelMatcher) String() string {
return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value) return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value)
} }

41
storage/metric/matcher_test.go
Unescape View File

@ -14,7 +14,12 @@
package metric package metric
import ( import (
"math/rand"
"reflect"
"sort"
"testing" "testing"
"github.com/prometheus/common/model"
) )
func TestAnchoredMatcher(t *testing.T) { func TestAnchoredMatcher(t *testing.T) {
@ -30,3 +35,39 @@ func TestAnchoredMatcher(t *testing.T) {
t.Errorf("Unexpected match for %q", "fooo") t.Errorf("Unexpected match for %q", "fooo")
} }
} }
func TestLabelMatchersSort(t *testing.T) {
// Line up Matchers in expected order:
want := LabelMatchers{
mustNewLabelMatcher(Equal, model.InstanceLabel, "not empty"),
mustNewLabelMatcher(Equal, model.MetricNameLabel, "a:recording:rule"),
mustNewLabelMatcher(Equal, model.MetricNameLabel, "up"),
mustNewLabelMatcher(Equal, "nothing_special but much longer", "not empty"),
mustNewLabelMatcher(Equal, "nothing_special", "not empty but longer"),
mustNewLabelMatcher(Equal, "nothing_special", "not empty"),
mustNewLabelMatcher(Equal, model.JobLabel, "not empty"),
mustNewLabelMatcher(Equal, model.BucketLabel, "not empty"),
mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string and is longer"),
mustNewLabelMatcher(RegexMatch, "irrelevant", "does not match empty string"),
mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string)?"),
mustNewLabelMatcher(RegexNoMatch, "irrelevant", "(matches empty string with a longer expression)?"),
mustNewLabelMatcher(NotEqual, "irrelevant", ""),
mustNewLabelMatcher(Equal, "irrelevant", ""),
}
got := make(LabelMatchers, len(want))
for i, j := range rand.Perm(len(want)) {
got[i] = want[j]
}
sort.Sort(got)
if !reflect.DeepEqual(want, got) {
t.Errorf("unexpected sorting of matchers, got %v, want %v", got, want)
}
}
func mustNewLabelMatcher(mt MatchType, name model.LabelName, val model.LabelValue) *LabelMatcher {
m, err := NewLabelMatcher(mt, name, val)
if err != nil {
panic(err)
}
return m
}

Write Preview
Loading…
Cancel
Save