|
|
@ -41,29 +41,26 @@ import ( |
|
|
|
const namespace = "prometheus" |
|
|
|
const namespace = "prometheus" |
|
|
|
|
|
|
|
|
|
|
|
var ( |
|
|
|
var ( |
|
|
|
evalDuration = prometheus.NewSummaryVec( |
|
|
|
evalDuration = prometheus.NewSummary( |
|
|
|
prometheus.SummaryOpts{ |
|
|
|
prometheus.SummaryOpts{ |
|
|
|
Namespace: namespace, |
|
|
|
Namespace: namespace, |
|
|
|
Name: "rule_evaluation_duration_seconds", |
|
|
|
Name: "rule_evaluation_duration_seconds", |
|
|
|
Help: "The duration for a rule to execute.", |
|
|
|
Help: "The duration for a rule to execute.", |
|
|
|
}, |
|
|
|
}, |
|
|
|
[]string{"rule_type"}, |
|
|
|
|
|
|
|
) |
|
|
|
) |
|
|
|
evalFailures = prometheus.NewCounterVec( |
|
|
|
evalFailures = prometheus.NewCounter( |
|
|
|
prometheus.CounterOpts{ |
|
|
|
prometheus.CounterOpts{ |
|
|
|
Namespace: namespace, |
|
|
|
Namespace: namespace, |
|
|
|
Name: "rule_evaluation_failures_total", |
|
|
|
Name: "rule_evaluation_failures_total", |
|
|
|
Help: "The total number of rule evaluation failures.", |
|
|
|
Help: "The total number of rule evaluation failures.", |
|
|
|
}, |
|
|
|
}, |
|
|
|
[]string{"rule_type"}, |
|
|
|
|
|
|
|
) |
|
|
|
) |
|
|
|
evalTotal = prometheus.NewCounterVec( |
|
|
|
evalTotal = prometheus.NewCounter( |
|
|
|
prometheus.CounterOpts{ |
|
|
|
prometheus.CounterOpts{ |
|
|
|
Namespace: namespace, |
|
|
|
Namespace: namespace, |
|
|
|
Name: "rule_evaluations_total", |
|
|
|
Name: "rule_evaluations_total", |
|
|
|
Help: "The total number of rule evaluations.", |
|
|
|
Help: "The total number of rule evaluations.", |
|
|
|
}, |
|
|
|
}, |
|
|
|
[]string{"rule_type"}, |
|
|
|
|
|
|
|
) |
|
|
|
) |
|
|
|
iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{ |
|
|
|
iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{ |
|
|
|
Namespace: namespace, |
|
|
|
Namespace: namespace, |
|
|
@ -96,11 +93,6 @@ var ( |
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
func init() { |
|
|
|
func init() { |
|
|
|
evalTotal.WithLabelValues(string(ruleTypeAlert)) |
|
|
|
|
|
|
|
evalTotal.WithLabelValues(string(ruleTypeRecording)) |
|
|
|
|
|
|
|
evalFailures.WithLabelValues(string(ruleTypeAlert)) |
|
|
|
|
|
|
|
evalFailures.WithLabelValues(string(ruleTypeRecording)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
prometheus.MustRegister(iterationDuration) |
|
|
|
prometheus.MustRegister(iterationDuration) |
|
|
|
prometheus.MustRegister(iterationsScheduled) |
|
|
|
prometheus.MustRegister(iterationsScheduled) |
|
|
|
prometheus.MustRegister(iterationsMissed) |
|
|
|
prometheus.MustRegister(iterationsMissed) |
|
|
@ -108,13 +100,6 @@ func init() { |
|
|
|
prometheus.MustRegister(evalDuration) |
|
|
|
prometheus.MustRegister(evalDuration) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
type ruleType string |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const ( |
|
|
|
|
|
|
|
ruleTypeAlert = "alerting" |
|
|
|
|
|
|
|
ruleTypeRecording = "recording" |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// QueryFunc processes PromQL queries.
|
|
|
|
// QueryFunc processes PromQL queries.
|
|
|
|
type QueryFunc func(ctx context.Context, q string, t time.Time) (promql.Vector, error) |
|
|
|
type QueryFunc func(ctx context.Context, q string, t time.Time) (promql.Vector, error) |
|
|
|
|
|
|
|
|
|
|
@ -329,16 +314,6 @@ func (g *Group) copyState(from *Group) { |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func typeForRule(r Rule) ruleType { |
|
|
|
|
|
|
|
switch r.(type) { |
|
|
|
|
|
|
|
case *AlertingRule: |
|
|
|
|
|
|
|
return ruleTypeAlert |
|
|
|
|
|
|
|
case *RecordingRule: |
|
|
|
|
|
|
|
return ruleTypeRecording |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
panic(fmt.Errorf("unknown rule type: %T", r)) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Eval runs a single evaluation cycle in which all rules are evaluated sequentially.
|
|
|
|
// Eval runs a single evaluation cycle in which all rules are evaluated sequentially.
|
|
|
|
func (g *Group) Eval(ctx context.Context, ts time.Time) { |
|
|
|
func (g *Group) Eval(ctx context.Context, ts time.Time) { |
|
|
|
for i, rule := range g.rules { |
|
|
|
for i, rule := range g.rules { |
|
|
@ -348,15 +323,13 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { |
|
|
|
default: |
|
|
|
default: |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
rtyp := string(typeForRule(rule)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func(i int, rule Rule) { |
|
|
|
func(i int, rule Rule) { |
|
|
|
defer func(t time.Time) { |
|
|
|
defer func(t time.Time) { |
|
|
|
evalDuration.WithLabelValues(rtyp).Observe(time.Since(t).Seconds()) |
|
|
|
evalDuration.Observe(time.Since(t).Seconds()) |
|
|
|
rule.SetEvaluationTime(time.Since(t)) |
|
|
|
rule.SetEvaluationTime(time.Since(t)) |
|
|
|
}(time.Now()) |
|
|
|
}(time.Now()) |
|
|
|
|
|
|
|
|
|
|
|
evalTotal.WithLabelValues(rtyp).Inc() |
|
|
|
evalTotal.Inc() |
|
|
|
|
|
|
|
|
|
|
|
vector, err := rule.Eval(ctx, ts, g.opts.QueryFunc, g.opts.ExternalURL) |
|
|
|
vector, err := rule.Eval(ctx, ts, g.opts.QueryFunc, g.opts.ExternalURL) |
|
|
|
if err != nil { |
|
|
|
if err != nil { |
|
|
@ -365,7 +338,7 @@ func (g *Group) Eval(ctx context.Context, ts time.Time) { |
|
|
|
if _, ok := err.(promql.ErrQueryCanceled); !ok { |
|
|
|
if _, ok := err.(promql.ErrQueryCanceled); !ok { |
|
|
|
level.Warn(g.logger).Log("msg", "Evaluating rule failed", "rule", rule, "err", err) |
|
|
|
level.Warn(g.logger).Log("msg", "Evaluating rule failed", "rule", rule, "err", err) |
|
|
|
} |
|
|
|
} |
|
|
|
evalFailures.WithLabelValues(rtyp).Inc() |
|
|
|
evalFailures.Inc() |
|
|
|
return |
|
|
|
return |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -645,13 +618,13 @@ func (m *Manager) AlertingRules() []*AlertingRule { |
|
|
|
return alerts |
|
|
|
return alerts |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Implements prometheus.Collector.
|
|
|
|
// Describe implements prometheus.Collector.
|
|
|
|
func (m *Manager) Describe(ch chan<- *prometheus.Desc) { |
|
|
|
func (m *Manager) Describe(ch chan<- *prometheus.Desc) { |
|
|
|
ch <- lastDuration |
|
|
|
ch <- lastDuration |
|
|
|
ch <- groupInterval |
|
|
|
ch <- groupInterval |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Implements prometheus.Collector.
|
|
|
|
// Collect implements prometheus.Collector.
|
|
|
|
func (m *Manager) Collect(ch chan<- prometheus.Metric) { |
|
|
|
func (m *Manager) Collect(ch chan<- prometheus.Metric) { |
|
|
|
for _, g := range m.RuleGroups() { |
|
|
|
for _, g := range m.RuleGroups() { |
|
|
|
ch <- prometheus.MustNewConstMetric(lastDuration, |
|
|
|
ch <- prometheus.MustNewConstMetric(lastDuration, |
|
|
|