github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Log rule name when evaluating rule groups' Eval function logs anything (#10454) 

* Add benchingmark test for rule group eval

Signed-off-by: Alvin Lin <alvinlin@amazon.com>
pull/10477/head
Alvin Lin 3 years ago committed by GitHub
parent
69e9eadfd3
commit
cd739214dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 9 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 18
      rules/manager.go

18
rules/manager.go
Unescape View File

@ -306,7 +306,7 @@ func NewGroup(o GroupOptions) *Group {
done: make(chan struct{}), done: make(chan struct{}),
managerDone: o.done, managerDone: o.done,
terminated: make(chan struct{}), terminated: make(chan struct{}),
logger: log.With(o.Opts.Logger, "group", o.Name), logger: log.With(o.Opts.Logger, "file", o.File, "group", o.Name),
metrics: metrics, metrics: metrics,
} }
} }
@ -607,7 +607,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
// Canceled queries are intentional termination of queries. This normally // Canceled queries are intentional termination of queries. This normally
// happens on shutdown and thus we skip logging of any errors here. // happens on shutdown and thus we skip logging of any errors here.
if _, ok := err.(promql.ErrQueryCanceled); !ok { if _, ok := err.(promql.ErrQueryCanceled); !ok {
level.Warn(g.logger).Log("msg", "Evaluating rule failed", "rule", rule, "err", err) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Evaluating rule failed", "rule", rule, "err", err)
} }
return return
} }
@ -631,7 +631,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
rule.SetLastError(err) rule.SetLastError(err)
g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc()
level.Warn(g.logger).Log("msg", "Rule sample appending failed", "err", err) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule sample appending failed", "err", err)
return return
} }
g.seriesInPreviousEval[i] = seriesReturned g.seriesInPreviousEval[i] = seriesReturned
@ -645,12 +645,12 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
switch errors.Cause(err) { switch errors.Cause(err) {
case storage.ErrOutOfOrderSample: case storage.ErrOutOfOrderSample:
numOutOfOrder++ numOutOfOrder++
level.Debug(g.logger).Log("msg", "Rule evaluation result discarded", "err", err, "sample", s) level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s)
case storage.ErrDuplicateSampleForTimestamp: case storage.ErrDuplicateSampleForTimestamp:
numDuplicates++ numDuplicates++
level.Debug(g.logger).Log("msg", "Rule evaluation result discarded", "err", err, "sample", s) level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s)
default: default:
level.Warn(g.logger).Log("msg", "Rule evaluation result discarded", "err", err, "sample", s) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s)
} }
} else { } else {
buf := [1024]byte{} buf := [1024]byte{}
@ -658,10 +658,10 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
} }
} }
if numOutOfOrder > 0 { if numOutOfOrder > 0 {
level.Warn(g.logger).Log("msg", "Error on ingesting out-of-order result from rule evaluation", "numDropped", numOutOfOrder) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting out-of-order result from rule evaluation", "numDropped", numOutOfOrder)
} }
if numDuplicates > 0 { if numDuplicates > 0 {
level.Warn(g.logger).Log("msg", "Error on ingesting results from rule evaluation with different value but same timestamp", "numDropped", numDuplicates) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting results from rule evaluation with different value but same timestamp", "numDropped", numDuplicates)
} }
for metric, lset := range g.seriesInPreviousEval[i] { for metric, lset := range g.seriesInPreviousEval[i] {
@ -674,7 +674,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) {
// Do not count these in logging, as this is expected if series // Do not count these in logging, as this is expected if series
// is exposed from a different rule. // is exposed from a different rule.
default: default:
level.Warn(g.logger).Log("msg", "Adding stale sample failed", "sample", lset.String(), "err", err) level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Adding stale sample failed", "sample", lset.String(), "err", err)
} }
} }
} }

Write Preview
Loading…
Cancel
Save