From 3a56817a30688ae37d3588369605390909874634 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Wed, 25 May 2022 10:06:17 +0200 Subject: [PATCH] Rules: set otel status to ERROR when a rule fails (#10745) Signed-off-by: Julien Pivotto --- rules/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rules/manager.go b/rules/manager.go index f8e88b510..4fdc13579 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -28,6 +28,7 @@ import ( "github.com/prometheus/common/model" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" @@ -624,6 +625,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { if err != nil { rule.SetHealth(HealthBad) rule.SetLastError(err) + sp.SetStatus(codes.Error, err.Error()) g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() // Canceled queries are intentional termination of queries. This normally @@ -651,6 +653,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { if err := app.Commit(); err != nil { rule.SetHealth(HealthBad) rule.SetLastError(err) + sp.SetStatus(codes.Error, err.Error()) g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule sample appending failed", "err", err) @@ -663,6 +666,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { if _, err := app.Append(0, s.Metric, s.T, s.V); err != nil { rule.SetHealth(HealthBad) rule.SetLastError(err) + sp.SetStatus(codes.Error, err.Error()) switch errors.Cause(err) { case storage.ErrOutOfOrderSample: