diff --git a/rules/alerting.go b/rules/alerting.go
index 785cb216f..861fdd99b 100644
--- a/rules/alerting.go
+++ b/rules/alerting.go
@@ -38,6 +38,17 @@ const (
// AlertState denotes the state of an active alert.
type AlertState int
+const (
+ // StateInactive is the state of an alert that is either firing nor pending.
+ StateInactive AlertState = iota
+ // StatePending is the state of an alert that has been active for less than
+ // the configured threshold duration.
+ StatePending
+ // StateFiring is the state of an alert that has been active for longer than
+ // the configured threshold duration.
+ StateFiring
+)
+
func (s AlertState) String() string {
switch s {
case StateInactive:
@@ -50,17 +61,13 @@ func (s AlertState) String() string {
panic(fmt.Errorf("unknown alert state: %v", s))
}
-const (
- StateInactive AlertState = iota
- StatePending
- StateFiring
-)
-
-type alertInstance struct {
- metric model.Metric
- value model.SampleValue
- state AlertState
- activeSince model.Time
+// Alert is the user-level representation of a single instance of an alerting rule.
+type Alert struct {
+ State AlertState
+ Labels model.LabelSet
+ Value model.SampleValue
+ ActiveAt model.Time
+ ResolvedAt model.Time
}
// An AlertingRule generates alerts from its vector expression.
@@ -81,7 +88,7 @@ type AlertingRule struct {
mtx sync.Mutex
// A map of alerts which are currently active (Pending or Firing), keyed by
// the fingerprint of the labelset they correspond to.
- active map[model.Fingerprint]*alertInstance
+ active map[model.Fingerprint]*Alert
}
// NewAlertingRule constructs a new AlertingRule.
@@ -92,7 +99,7 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann
holdDuration: hold,
labels: lbls,
annotations: anns,
- active: map[model.Fingerprint]*alertInstance{},
+ active: map[model.Fingerprint]*Alert{},
}
}
@@ -101,17 +108,17 @@ func (rule *AlertingRule) Name() string {
return rule.name
}
-func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model.Sample {
+func (r *AlertingRule) sample(alert *Alert, ts model.Time, set bool) *model.Sample {
// Build alert labels in order they can be overwritten.
metric := model.Metric(r.labels.Clone())
- for ln, lv := range ai.metric {
+ for ln, lv := range alert.Labels {
metric[ln] = lv
}
metric[model.MetricNameLabel] = alertMetricName
metric[model.AlertNameLabel] = model.LabelValue(r.name)
- metric[alertStateLabel] = model.LabelValue(ai.state.String())
+ metric[alertStateLabel] = model.LabelValue(alert.State.String())
s := &model.Sample{
Metric: metric,
@@ -124,6 +131,10 @@ func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model
return s
}
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequentally repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
// eval evaluates the rule expression and then creates pending alerts and fires
// or removes previously pending alerts accordingly.
func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) {
@@ -147,57 +158,59 @@ func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector,
fp := smpl.Metric.Fingerprint()
resultFPs[fp] = struct{}{}
- if ai, ok := r.active[fp]; ok {
- ai.value = smpl.Value
+ if alert, ok := r.active[fp]; ok {
+ alert.Value = smpl.Value
continue
}
delete(smpl.Metric, model.MetricNameLabel)
- r.active[fp] = &alertInstance{
- metric: smpl.Metric,
- activeSince: ts,
- state: StatePending,
- value: smpl.Value,
+ r.active[fp] = &Alert{
+ Labels: model.LabelSet(smpl.Metric),
+ ActiveAt: ts,
+ State: StatePending,
+ Value: smpl.Value,
}
}
var vec model.Vector
// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
- for fp, ai := range r.active {
+ for fp, a := range r.active {
if _, ok := resultFPs[fp]; !ok {
- delete(r.active, fp)
- vec = append(vec, r.sample(ai, ts, false))
+ if a.State != StateInactive {
+ vec = append(vec, r.sample(a, ts, false))
+ }
+ // If the alert was previously firing, keep it aroud for a given
+ // retention time so it is reported as resolved to the AlertManager.
+ if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) {
+ delete(r.active, fp)
+ }
+ if a.State != StateInactive {
+ a.State = StateInactive
+ a.ResolvedAt = ts
+ }
continue
}
- if ai.state != StateFiring && ts.Sub(ai.activeSince) >= r.holdDuration {
- vec = append(vec, r.sample(ai, ts, false))
- ai.state = StateFiring
+ if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration {
+ vec = append(vec, r.sample(a, ts, false))
+ a.State = StateFiring
}
- vec = append(vec, r.sample(ai, ts, true))
+ vec = append(vec, r.sample(a, ts, true))
}
return vec, nil
}
-// Alert is the user-level representation of a single instance of an alerting rule.
-type Alert struct {
- State AlertState
- Labels model.LabelSet
- ActiveSince model.Time
- Value model.SampleValue
-}
-
func (r *AlertingRule) State() AlertState {
r.mtx.Lock()
defer r.mtx.Unlock()
maxState := StateInactive
- for _, ai := range r.active {
- if ai.state > maxState {
- maxState = ai.state
+ for _, a := range r.active {
+ if a.State > maxState {
+ maxState = a.State
}
}
return maxState
@@ -205,21 +218,30 @@ func (r *AlertingRule) State() AlertState {
// ActiveAlerts returns a slice of active alerts.
func (r *AlertingRule) ActiveAlerts() []*Alert {
+ var res []*Alert
+ for _, a := range r.recentAlerts() {
+ if a.ResolvedAt == 0 {
+ res = append(res, a)
+ }
+ }
+ return res
+}
+
+func (r *AlertingRule) recentAlerts() []*Alert {
r.mtx.Lock()
defer r.mtx.Unlock()
alerts := make([]*Alert, 0, len(r.active))
- for _, ai := range r.active {
+
+ for _, a := range r.active {
labels := r.labels.Clone()
- for ln, lv := range ai.metric {
+ for ln, lv := range a.Labels {
labels[ln] = lv
}
- alerts = append(alerts, &Alert{
- State: ai.state,
- Labels: labels,
- ActiveSince: ai.activeSince,
- Value: ai.value,
- })
+ anew := *a
+ anew.Labels = labels
+
+ alerts = append(alerts, &anew)
}
return alerts
}
diff --git a/rules/manager.go b/rules/manager.go
index 352e17837..1ff9eefe6 100644
--- a/rules/manager.go
+++ b/rules/manager.go
@@ -241,9 +241,10 @@ func (g *Group) eval() {
func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
var alerts model.Alerts
- for _, alert := range rule.ActiveAlerts() {
+
+ for _, alert := range rule.recentAlerts() {
// Only send actually firing alerts.
- if alert.State != StateFiring {
+ if alert.State == StatePending {
continue
}
@@ -292,12 +293,17 @@ func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
annotations[an] = expand(av)
}
- alerts = append(alerts, &model.Alert{
- StartsAt: alert.ActiveSince.Time().Add(rule.holdDuration),
+ a := &model.Alert{
+ StartsAt: alert.ActiveAt.Add(rule.holdDuration).Time(),
Labels: labels,
Annotations: annotations,
GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
- })
+ }
+ if alert.ResolvedAt != 0 {
+ a.EndsAt = alert.ResolvedAt.Time()
+ }
+
+ alerts = append(alerts, a)
}
if len(alerts) > 0 {
diff --git a/web/ui/bindata.go b/web/ui/bindata.go
index 1acfe71d2..92ccc2bd5 100644
--- a/web/ui/bindata.go
+++ b/web/ui/bindata.go
@@ -119,7 +119,7 @@ func webUiTemplates_baseHtml() (*asset, error) {
return a, nil
}
-var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\x66\xde\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\x63\x72\x2a\x41\x5b\x93\x79\x78\xe9\x79\x0d\xd3\xa2\xb5\xc4\x68\xb6\x8c\x7b\xbb\xc0\xde\x10\x55\xe4\x11\x53\xde\x39\x07\x8a\xa3\xbc\xbb\x0f\xc5\xac\x51\x16\x94\xf5\xa2\x0b\x2e\x8e\x84\x49\x6a\xcc\x2a\x98\x29\x06\xe8\x74\x27\x0f\x82\xc7\xab\xeb\xaf\xe5\x43\xa0\x2d\x72\xfc\xf4\x16\x4b\x2b\x09\x03\x26\x1e\xc2\x6f\x5a\x35\x9a\x83\x06\xde\x1f\x59\x23\x25\x6d\x0d\x44\x22\x0f\xac\x1a\x7e\x8e\xdf\xce\xdd\x07\xb1\x6b\xd4\x0e\x9b\xe6\xb9\x79\x7f\xf4\x7c\xe4\xdb\x8a\x64\x0f\x33\x8e\x90\x5e\x0f\xd3\x54\xbd\x42\x1f\x23\xd4\xeb\xf3\x01\xb3\xda\x3b\x23\x2b\xb3\xe2\x08\x51\x71\x64\x1b\x19\x2e\x81\x85\xd5\xc3\x03\x9c\x13\x8a\xc3\x89\xcc\xeb\xc9\x82\xa1\xeb\x48\xf0\x6e\x7d\xa9\x41\xf7\xef\x89\x44\xbc\x2c\xc4\xc0\x25\x30\x83\x29\xab\xe1\xa8\xf1\x9f\x37\xef\xca\xd7\x41\x94\xa4\xa8\x4a\xe7\xb2\x9f\x74\x8f\x4c\x45\x5e\x95\xe4\x93\x73\x12\x14\xb9\x52\xeb\x2f\x09\xc7\xcf\x45\x8e\xac\x83\xd2\xdc\xea\xf2\x56\x75\x94\xc3\x01\xeb\x25\xcd\x44\xcf\xe5\x80\x47\xac\xee\xf8\x8c\x96\x56\x43\x59\xb0\x86\x83\x97\xf4\x7d\xf3\xe3\x69\xad\x44\xdb\x82\x1d\x35\x95\x17\x19\x22\x8a\xdc\x47\x8f\xf9\xf2\x09\x21\x66\x6f\x37\x7d\xc6\x38\xfe\x7f\x7b\xa5\x6e\x8e\xa0\x2f\x7d\x83\x05\x51\xd8\x37\x7d\xd2\x41\xc2\x1e\xbb\xd5\x6c\x83\x3b\x99\xbc\xe7\x23\x27\x13\x8f\xf7\xd5\xe5\x13\xad\x40\x62\xef\xe2\xe7\x8c\x37\x54\x77\xc9\x19\x3b\x87\xac\x85\x62\x8b\x31\x2f\x54\x1e\x16\x9d\x6b\x81\x45\x9e\xc3\x8e\x8b\x3a\xe4\x31\x36\xf6\x72\x2a\xc3\x53\x6f\x6f\xe1\x53\xd3\x88\x4b\xfa\xb7\x7f\x21\xf7\x47\x2f\x32\x0c\x43\xcc\xc6\x84\xb7\xa7\x32\x2d\x55\x43\x2a\x03\x92\x84\xdf\xb4\xd5\x62\x4f\xf5\x39\xc1\x7e\x89\x8c\x5d\xe7\xa7\x26\xb2\x76\x5d\x82\x9b\x06\x91\x73\x32\xe2\xde\x99\x5c\x93\xdf\x4a\x0e\x43\x34\xbe\x3e\xd4\x3d\x56\x3f\xc5\x75\x17\x87\x90\xfc\x25\xe3\x11\x8d\xf3\x89\x43\xe3\xd7\x1f\x6c\x71\x88\x05\xa3\xb6\xc1\x26\xc2\xc5\x9b\x1e\xb0\xa5\x35\xa3\x06\xbc\xec\x61\x88\x7b\xa5\x4b\x12\x30\x30\x96\x3c\x54\x3c\xdb\x88\x3d\x64\xbf\x37\x8f\x1e\xb7\x08\x78\x89\x49\xb8\x8d\x98\x2b\xf1\x34\x1f\x18\xe3\x3b\xfa\x7a\x9e\xae\x83\xe6\x57\xc1\x38\x0a\xad\xc3\x5a\xbd\xf0\xf5\x63\x3a\x84\xfd\x0b\x00\x00\xff\xff\xdb\x8e\x0c\x63\xab\x06\x00\x00")
+var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\xe6\xcd\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\xc7\xe4\x54\x82\xb6\x26\xf3\xe1\xa5\xe7\x35\x4c\x8b\xd6\x12\xa3\xd9\x72\xdc\xdb\x25\xec\x0d\xa3\x8a\x3c\xc6\x94\x77\xce\x81\xe2\x28\xef\xee\x43\x31\x6b\x94\x05\x65\xbd\xe8\x82\x8b\x23\x61\x92\x1a\xb3\x0a\x66\x8a\x00\x9d\xee\xe4\x41\xf0\x78\x75\xfd\xb5\x7c\x08\xb4\x45\x8e\x9f\xde\x62\x69\x25\x61\x88\x89\x87\xf0\x9b\x56\x8d\xe6\xa0\x81\xf7\x47\xd6\x48\x49\x5b\x03\x91\xc8\x07\x56\x0d\x3f\xc7\x6f\xe7\xee\x83\xd8\x35\x6a\x87\x4d\xf3\xdc\xbc\x3f\x7a\x3e\xf2\x6d\x45\xb2\x87\x19\x47\x48\xaf\x0f\xd3\x54\xbd\x42\x8f\x11\xea\xf5\xf9\x80\x59\xed\x9d\x91\x95\x59\x71\x84\xa8\x38\xb2\x8d\x0c\x17\x60\x61\xf5\xf0\x00\xe7\x84\xe2\x70\x22\xf3\x7a\xb2\x60\xe8\x3a\x12\xbc\x5b\x5f\x6a\xd0\xfd\x7b\x22\x11\x2f\x0b\x31\x70\x09\xcc\x60\xca\x6a\x38\x6a\xfc\xe7\xcd\xbb\xf2\x75\x10\x25\x29\xaa\xd2\xb9\xec\x27\xdd\x23\x53\x91\x57\x25\xf9\xe4\x9c\x04\x45\xae\xd4\xfa\x4b\xc2\xf1\x73\x91\x23\xeb\xa0\x34\xb7\xba\xbc\x55\x1d\xe5\x70\xc0\x7a\x49\x33\xd1\x73\x39\xe0\x11\xab\x3b\x3e\xa3\xa5\xd5\x50\x16\xac\xe1\xe0\x25\x7d\xdf\xfc\x78\x5a\x2b\xd1\xb6\x60\x47\x4d\xe5\x45\x06\x44\x91\x7b\xf4\x98\x2f\x9f\x10\x62\xf6\x76\xd3\x67\x8c\xf1\xff\xdb\x2b\x75\x73\x04\x7d\xe9\x1b\x2c\x88\xc2\xbe\xe9\x93\x0e\x12\xf6\xd8\xad\x66\x1b\xdc\xc9\xe4\x3d\x1f\x39\x99\x78\xbc\xaf\x2e\x9f\x68\x05\x12\x7b\x17\x3f\x67\xbc\xa1\xba\x4b\xce\xd8\x39\x64\x2d\x14\x5b\xc4\xbc\x50\x79\x58\x74\xae\x05\x16\x79\x2e\x76\x5c\xd4\x21\x8f\xb1\xb1\x97\x53\x19\x9e\x7a\x7b\x0b\x9f\x9a\x46\x5c\xd2\xbf\xfd\x0b\xb9\x3f\x7a\x91\x61\x18\x62\x36\x26\xbc\x3d\x95\x69\xa9\x1a\x52\x19\x22\x49\xf8\x4d\x5b\x2d\xf6\x54\x9f\x13\xec\x97\xc8\xd8\x75\x7e\x6a\x22\x6b\xd7\x25\xb8\x69\x30\x72\x4e\x46\xdc\x3b\x93\x6b\xf2\x5b\xc9\x61\x88\xc6\xd7\x87\xba\xc7\xea\xa7\xb8\xee\xe2\x10\x92\xbf\x64\x3c\xa2\x71\x3e\x71\x68\xfc\xfa\x83\x2d\x0e\xb1\x60\xd4\x36\xd8\x44\xb8\x78\xd3\x03\xb6\xb4\x66\xd4\x80\x97\x3d\x0c\x71\xaf\x74\x49\x02\x02\xfb\x65\x61\xb3\x8d\xd8\x43\xf6\x7b\xf3\xe8\x83\x16\xd1\x2f\x31\x03\xb7\x88\xb9\xfa\x4e\x93\x81\x18\xdf\xce\xd7\xc3\x74\x0d\x9a\xdf\x03\x63\x14\x5a\x87\x9d\x7a\xe1\xeb\x67\x74\x80\xfd\x0b\x00\x00\xff\xff\xb3\xf6\xaa\xca\xa8\x06\x00\x00")
func webUiTemplatesAlertsHtmlBytes() ([]byte, error) {
return bindataRead(
@@ -134,7 +134,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) {
return nil, err
}
- info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450348618, 0)}
+ info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1704, mode: os.FileMode(420), modTime: time.Unix(1450348695, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
diff --git a/web/ui/templates/alerts.html b/web/ui/templates/alerts.html
index 58a00be34..4bdaec5f1 100644
--- a/web/ui/templates/alerts.html
+++ b/web/ui/templates/alerts.html
@@ -36,7 +36,7 @@
{{end}}
| {{.State}} |
- {{.ActiveSince.Time.UTC}} |
+ {{.ActiveAt.Time.UTC}} |
{{.Value}} |
{{end}}