mirror of https://github.com/prometheus/prometheus
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
339 lines
8.9 KiB
339 lines
8.9 KiB
// Copyright 2013 The Prometheus Authors |
|
// Licensed under the Apache License, Version 2.0 (the "License"); |
|
// you may not use this file except in compliance with the License. |
|
// You may obtain a copy of the License at |
|
// |
|
// http://www.apache.org/licenses/LICENSE-2.0 |
|
// |
|
// Unless required by applicable law or agreed to in writing, software |
|
// distributed under the License is distributed on an "AS IS" BASIS, |
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
// See the License for the specific language governing permissions and |
|
// limitations under the License. |
|
|
|
package rules |
|
|
|
import ( |
|
"fmt" |
|
"io/ioutil" |
|
"sync" |
|
"time" |
|
|
|
"github.com/golang/glog" |
|
"github.com/prometheus/client_golang/prometheus" |
|
|
|
clientmodel "github.com/prometheus/client_golang/model" |
|
|
|
"github.com/prometheus/prometheus/config" |
|
"github.com/prometheus/prometheus/notification" |
|
"github.com/prometheus/prometheus/promql" |
|
"github.com/prometheus/prometheus/storage" |
|
"github.com/prometheus/prometheus/templates" |
|
"github.com/prometheus/prometheus/utility" |
|
) |
|
|
|
// Constants for instrumentation. |
|
const ( |
|
namespace = "prometheus" |
|
|
|
ruleTypeLabel = "rule_type" |
|
alertingRuleType = "alerting" |
|
recordingRuleType = "recording" |
|
) |
|
|
|
var ( |
|
evalDuration = prometheus.NewSummaryVec( |
|
prometheus.SummaryOpts{ |
|
Namespace: namespace, |
|
Name: "rule_evaluation_duration_milliseconds", |
|
Help: "The duration for a rule to execute.", |
|
}, |
|
[]string{ruleTypeLabel}, |
|
) |
|
evalFailures = prometheus.NewCounter( |
|
prometheus.CounterOpts{ |
|
Namespace: namespace, |
|
Name: "rule_evaluation_failures_total", |
|
Help: "The total number of rule evaluation failures.", |
|
}, |
|
) |
|
iterationDuration = prometheus.NewSummary(prometheus.SummaryOpts{ |
|
Namespace: namespace, |
|
Name: "evaluator_duration_milliseconds", |
|
Help: "The duration for all evaluations to execute.", |
|
Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, |
|
}) |
|
) |
|
|
|
func init() { |
|
prometheus.MustRegister(iterationDuration) |
|
prometheus.MustRegister(evalFailures) |
|
prometheus.MustRegister(evalDuration) |
|
} |
|
|
|
// The Manager manages recording and alerting rules. |
|
type Manager struct { |
|
// Protects the rules list. |
|
sync.Mutex |
|
rules []Rule |
|
|
|
done chan bool |
|
|
|
interval time.Duration |
|
queryEngine *promql.Engine |
|
|
|
sampleAppender storage.SampleAppender |
|
notificationHandler *notification.NotificationHandler |
|
|
|
prometheusURL string |
|
pathPrefix string |
|
} |
|
|
|
// ManagerOptions bundles options for the Manager. |
|
type ManagerOptions struct { |
|
EvaluationInterval time.Duration |
|
QueryEngine *promql.Engine |
|
|
|
NotificationHandler *notification.NotificationHandler |
|
SampleAppender storage.SampleAppender |
|
|
|
PrometheusURL string |
|
PathPrefix string |
|
} |
|
|
|
// NewManager returns an implementation of Manager, ready to be started |
|
// by calling the Run method. |
|
func NewManager(o *ManagerOptions) *Manager { |
|
manager := &Manager{ |
|
rules: []Rule{}, |
|
done: make(chan bool), |
|
|
|
interval: o.EvaluationInterval, |
|
sampleAppender: o.SampleAppender, |
|
queryEngine: o.QueryEngine, |
|
notificationHandler: o.NotificationHandler, |
|
prometheusURL: o.PrometheusURL, |
|
} |
|
return manager |
|
} |
|
|
|
// Run the rule manager's periodic rule evaluation. |
|
func (m *Manager) Run() { |
|
defer glog.Info("Rule manager stopped.") |
|
|
|
m.Lock() |
|
lastInterval := m.interval |
|
m.Unlock() |
|
|
|
ticker := time.NewTicker(lastInterval) |
|
defer ticker.Stop() |
|
|
|
for { |
|
// The outer select clause makes sure that m.done is looked at |
|
// first. Otherwise, if m.runIteration takes longer than |
|
// m.interval, there is only a 50% chance that m.done will be |
|
// looked at before the next m.runIteration call happens. |
|
select { |
|
case <-m.done: |
|
return |
|
default: |
|
select { |
|
case <-ticker.C: |
|
start := time.Now() |
|
m.runIteration() |
|
iterationDuration.Observe(float64(time.Since(start) / time.Millisecond)) |
|
|
|
m.Lock() |
|
if lastInterval != m.interval { |
|
ticker.Stop() |
|
ticker = time.NewTicker(m.interval) |
|
lastInterval = m.interval |
|
} |
|
m.Unlock() |
|
case <-m.done: |
|
return |
|
} |
|
} |
|
} |
|
} |
|
|
|
// Stop the rule manager's rule evaluation cycles. |
|
func (m *Manager) Stop() { |
|
glog.Info("Stopping rule manager...") |
|
m.done <- true |
|
} |
|
|
|
func (m *Manager) queueAlertNotifications(rule *AlertingRule, timestamp clientmodel.Timestamp) { |
|
activeAlerts := rule.ActiveAlerts() |
|
if len(activeAlerts) == 0 { |
|
return |
|
} |
|
|
|
notifications := make(notification.NotificationReqs, 0, len(activeAlerts)) |
|
for _, aa := range activeAlerts { |
|
if aa.State != Firing { |
|
// BUG: In the future, make AlertManager support pending alerts? |
|
continue |
|
} |
|
|
|
// Provide the alert information to the template. |
|
l := map[string]string{} |
|
for k, v := range aa.Labels { |
|
l[string(k)] = string(v) |
|
} |
|
tmplData := struct { |
|
Labels map[string]string |
|
Value clientmodel.SampleValue |
|
}{ |
|
Labels: l, |
|
Value: aa.Value, |
|
} |
|
// Inject some convenience variables that are easier to remember for users |
|
// who are not used to Go's templating system. |
|
defs := "{{$labels := .Labels}}{{$value := .Value}}" |
|
|
|
expand := func(text string) string { |
|
template := templates.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.pathPrefix) |
|
result, err := template.Expand() |
|
if err != nil { |
|
result = err.Error() |
|
glog.Warningf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err) |
|
} |
|
return result |
|
} |
|
|
|
notifications = append(notifications, ¬ification.NotificationReq{ |
|
Summary: expand(rule.Summary), |
|
Description: expand(rule.Description), |
|
Labels: aa.Labels.Merge(clientmodel.LabelSet{ |
|
AlertNameLabel: clientmodel.LabelValue(rule.Name()), |
|
}), |
|
Value: aa.Value, |
|
ActiveSince: aa.ActiveSince.Time(), |
|
RuleString: rule.String(), |
|
GeneratorURL: m.prometheusURL + utility.GraphLinkForExpression(rule.Vector.String()), |
|
}) |
|
} |
|
m.notificationHandler.SubmitReqs(notifications) |
|
} |
|
|
|
func (m *Manager) runIteration() { |
|
now := clientmodel.Now() |
|
wg := sync.WaitGroup{} |
|
|
|
m.Lock() |
|
rulesSnapshot := make([]Rule, len(m.rules)) |
|
copy(rulesSnapshot, m.rules) |
|
m.Unlock() |
|
|
|
for _, rule := range rulesSnapshot { |
|
wg.Add(1) |
|
// BUG(julius): Look at fixing thundering herd. |
|
go func(rule Rule) { |
|
defer wg.Done() |
|
|
|
start := time.Now() |
|
vector, err := rule.Eval(now, m.queryEngine) |
|
duration := time.Since(start) |
|
|
|
if err != nil { |
|
evalFailures.Inc() |
|
glog.Warningf("Error while evaluating rule %q: %s", rule, err) |
|
return |
|
} |
|
|
|
switch r := rule.(type) { |
|
case *AlertingRule: |
|
m.queueAlertNotifications(r, now) |
|
evalDuration.WithLabelValues(alertingRuleType).Observe( |
|
float64(duration / time.Millisecond), |
|
) |
|
case *RecordingRule: |
|
evalDuration.WithLabelValues(recordingRuleType).Observe( |
|
float64(duration / time.Millisecond), |
|
) |
|
default: |
|
panic(fmt.Errorf("Unknown rule type: %T", rule)) |
|
} |
|
|
|
for _, s := range vector { |
|
m.sampleAppender.Append(&clientmodel.Sample{ |
|
Metric: s.Metric.Metric, |
|
Value: s.Value, |
|
Timestamp: s.Timestamp, |
|
}) |
|
} |
|
}(rule) |
|
} |
|
wg.Wait() |
|
} |
|
|
|
// ApplyConfig updates the rule manager's state as the config requires. If |
|
// loading the new rules failed the old rule set is restored. |
|
func (m *Manager) ApplyConfig(conf *config.Config) { |
|
m.Lock() |
|
defer m.Unlock() |
|
|
|
m.interval = time.Duration(conf.GlobalConfig.EvaluationInterval) |
|
|
|
rulesSnapshot := make([]Rule, len(m.rules)) |
|
copy(rulesSnapshot, m.rules) |
|
m.rules = m.rules[:0] |
|
|
|
if err := m.loadRuleFiles(conf.RuleFiles...); err != nil { |
|
// If loading the new rules failed, restore the old rule set. |
|
m.rules = rulesSnapshot |
|
glog.Errorf("Error loading rules, previous rule set restored: %s", err) |
|
} |
|
} |
|
|
|
// loadRuleFiles loads alerting and recording rules from the given files. |
|
func (m *Manager) loadRuleFiles(filenames ...string) error { |
|
for _, fn := range filenames { |
|
content, err := ioutil.ReadFile(fn) |
|
if err != nil { |
|
return err |
|
} |
|
stmts, err := promql.ParseStmts(string(content)) |
|
if err != nil { |
|
return fmt.Errorf("error parsing %s: %s", fn, err) |
|
} |
|
for _, stmt := range stmts { |
|
switch r := stmt.(type) { |
|
case *promql.AlertStmt: |
|
rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description) |
|
m.rules = append(m.rules, rule) |
|
case *promql.RecordStmt: |
|
rule := &RecordingRule{r.Name, r.Expr, r.Labels} |
|
m.rules = append(m.rules, rule) |
|
default: |
|
panic("retrieval.Manager.LoadRuleFiles: unknown statement type") |
|
} |
|
} |
|
} |
|
return nil |
|
} |
|
|
|
// Rules returns the list of the manager's rules. |
|
func (m *Manager) Rules() []Rule { |
|
m.Lock() |
|
defer m.Unlock() |
|
|
|
rules := make([]Rule, len(m.rules)) |
|
copy(rules, m.rules) |
|
return rules |
|
} |
|
|
|
// AlertingRules returns the list of the manager's alerting rules. |
|
func (m *Manager) AlertingRules() []*AlertingRule { |
|
m.Lock() |
|
defer m.Unlock() |
|
|
|
alerts := []*AlertingRule{} |
|
for _, rule := range m.rules { |
|
if alertingRule, ok := rule.(*AlertingRule); ok { |
|
alerts = append(alerts, alertingRule) |
|
} |
|
} |
|
return alerts |
|
}
|
|
|