Send resolved notifications

rules/alerting.go

@@ -38,6 +38,17 @@ const (
 // AlertState denotes the state of an active alert.
 type AlertState int
 
+const (
+	// StateInactive is the state of an alert that is either firing nor pending.
+	StateInactive AlertState = iota
+	// StatePending is the state of an alert that has been active for less than
+	// the configured threshold duration.
+	StatePending
+	// StateFiring is the state of an alert that has been active for longer than
+	// the configured threshold duration.
+	StateFiring
+)
+
 func (s AlertState) String() string {
 	switch s {
 	case StateInactive:
@@ -50,17 +61,13 @@ func (s AlertState) String() string {
 	panic(fmt.Errorf("unknown alert state: %v", s))
 }
 
-const (
-	StateInactive AlertState = iota
-	StatePending
-	StateFiring
-)
-
-type alertInstance struct {
-	metric      model.Metric
-	value       model.SampleValue
-	state       AlertState
-	activeSince model.Time
+// Alert is the user-level representation of a single instance of an alerting rule.
+type Alert struct {
+	State      AlertState
+	Labels     model.LabelSet
+	Value      model.SampleValue
+	ActiveAt   model.Time
+	ResolvedAt model.Time
 }
 
 // An AlertingRule generates alerts from its vector expression.
@@ -81,7 +88,7 @@ type AlertingRule struct {
 	mtx sync.Mutex
 	// A map of alerts which are currently active (Pending or Firing), keyed by
 	// the fingerprint of the labelset they correspond to.
-	active map[model.Fingerprint]*alertInstance
+	active map[model.Fingerprint]*Alert
 }
 
 // NewAlertingRule constructs a new AlertingRule.
@@ -92,7 +99,7 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann
 		holdDuration: hold,
 		labels:       lbls,
 		annotations:  anns,
-		active:       map[model.Fingerprint]*alertInstance{},
+		active:       map[model.Fingerprint]*Alert{},
 	}
 }
 
@@ -101,17 +108,17 @@ func (rule *AlertingRule) Name() string {
 	return rule.name
 }
 
-func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model.Sample {
+func (r *AlertingRule) sample(alert *Alert, ts model.Time, set bool) *model.Sample {
 	// Build alert labels in order they can be overwritten.
 	metric := model.Metric(r.labels.Clone())
 
-	for ln, lv := range ai.metric {
+	for ln, lv := range alert.Labels {
 		metric[ln] = lv
 	}
 
 	metric[model.MetricNameLabel] = alertMetricName
 	metric[model.AlertNameLabel] = model.LabelValue(r.name)
-	metric[alertStateLabel] = model.LabelValue(ai.state.String())
+	metric[alertStateLabel] = model.LabelValue(alert.State.String())
 
 	s := &model.Sample{
 		Metric:    metric,
@@ -124,6 +131,10 @@ func (r *AlertingRule) sample(ai *alertInstance, ts model.Time, set bool) *model
 	return s
 }
 
+// resolvedRetention is the duration for which a resolved alert instance
+// is kept in memory state and consequentally repeatedly sent to the AlertManager.
+const resolvedRetention = 15 * time.Minute
+
 // eval evaluates the rule expression and then creates pending alerts and fires
 // or removes previously pending alerts accordingly.
 func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector, error) {
@@ -147,57 +158,59 @@ func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine) (model.Vector,
 		fp := smpl.Metric.Fingerprint()
 		resultFPs[fp] = struct{}{}
 
-		if ai, ok := r.active[fp]; ok {
-			ai.value = smpl.Value
+		if alert, ok := r.active[fp]; ok {
+			alert.Value = smpl.Value
 			continue
 		}
 
 		delete(smpl.Metric, model.MetricNameLabel)
 
-		r.active[fp] = &alertInstance{
-			metric:      smpl.Metric,
-			activeSince: ts,
-			state:       StatePending,
-			value:       smpl.Value,
+		r.active[fp] = &Alert{
+			Labels:   model.LabelSet(smpl.Metric),
+			ActiveAt: ts,
+			State:    StatePending,
+			Value:    smpl.Value,
 		}
 	}
 
 	var vec model.Vector
 	// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
-	for fp, ai := range r.active {
+	for fp, a := range r.active {
 		if _, ok := resultFPs[fp]; !ok {
-			delete(r.active, fp)
-			vec = append(vec, r.sample(ai, ts, false))
+			if a.State != StateInactive {
+				vec = append(vec, r.sample(a, ts, false))
+			}
+			// If the alert was previously firing, keep it aroud for a given
+			// retention time so it is reported as resolved to the AlertManager.
+			if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) {
+				delete(r.active, fp)
+			}
+			if a.State != StateInactive {
+				a.State = StateInactive
+				a.ResolvedAt = ts
+			}
 			continue
 		}
 
-		if ai.state != StateFiring && ts.Sub(ai.activeSince) >= r.holdDuration {
-			vec = append(vec, r.sample(ai, ts, false))
-			ai.state = StateFiring
+		if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration {
+			vec = append(vec, r.sample(a, ts, false))
+			a.State = StateFiring
 		}
 
-		vec = append(vec, r.sample(ai, ts, true))
+		vec = append(vec, r.sample(a, ts, true))
 	}
 
 	return vec, nil
 }
 
-// Alert is the user-level representation of a single instance of an alerting rule.
-type Alert struct {
-	State       AlertState
-	Labels      model.LabelSet
-	ActiveSince model.Time
-	Value       model.SampleValue
-}
-
 func (r *AlertingRule) State() AlertState {
 	r.mtx.Lock()
 	defer r.mtx.Unlock()
 
 	maxState := StateInactive
-	for _, ai := range r.active {
-		if ai.state > maxState {
-			maxState = ai.state
+	for _, a := range r.active {
+		if a.State > maxState {
+			maxState = a.State
 		}
 	}
 	return maxState
@@ -205,21 +218,30 @@ func (r *AlertingRule) State() AlertState {
 
 // ActiveAlerts returns a slice of active alerts.
 func (r *AlertingRule) ActiveAlerts() []*Alert {
+	var res []*Alert
+	for _, a := range r.recentAlerts() {
+		if a.ResolvedAt == 0 {
+			res = append(res, a)
+		}
+	}
+	return res
+}
+
+func (r *AlertingRule) recentAlerts() []*Alert {
 	r.mtx.Lock()
 	defer r.mtx.Unlock()
 
 	alerts := make([]*Alert, 0, len(r.active))
-	for _, ai := range r.active {
+
+	for _, a := range r.active {
 		labels := r.labels.Clone()
-		for ln, lv := range ai.metric {
+		for ln, lv := range a.Labels {
 			labels[ln] = lv
 		}
-		alerts = append(alerts, &Alert{
-			State:       ai.state,
-			Labels:      labels,
-			ActiveSince: ai.activeSince,
-			Value:       ai.value,
-		})
+		anew := *a
+		anew.Labels = labels
+
+		alerts = append(alerts, &anew)
 	}
 	return alerts
 }

rules/manager.go

@@ -241,9 +241,10 @@ func (g *Group) eval() {
 
 func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
 	var alerts model.Alerts
-	for _, alert := range rule.ActiveAlerts() {
+
+	for _, alert := range rule.recentAlerts() {
 		// Only send actually firing alerts.
-		if alert.State != StateFiring {
+		if alert.State == StatePending {
 			continue
 		}
 
@@ -292,12 +293,17 @@ func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error {
 			annotations[an] = expand(av)
 		}
 
-		alerts = append(alerts, &model.Alert{
-			StartsAt:     alert.ActiveSince.Time().Add(rule.holdDuration),
+		a := &model.Alert{
+			StartsAt:     alert.ActiveAt.Add(rule.holdDuration).Time(),
 			Labels:       labels,
 			Annotations:  annotations,
 			GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
-		})
+		}
+		if alert.ResolvedAt != 0 {
+			a.EndsAt = alert.ResolvedAt.Time()
+		}
+
+		alerts = append(alerts, a)
 	}
 
 	if len(alerts) > 0 {

web/ui/bindata.go

@@ -119,7 +119,7 @@ func webUiTemplates_baseHtml() (*asset, error) {
 	return a, nil
 }
 
-var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\x66\xde\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\x63\x72\x2a\x41\x5b\x93\x79\x78\xe9\x79\x0d\xd3\xa2\xb5\xc4\x68\xb6\x8c\x7b\xbb\xc0\xde\x10\x55\xe4\x11\x53\xde\x39\x07\x8a\xa3\xbc\xbb\x0f\xc5\xac\x51\x16\x94\xf5\xa2\x0b\x2e\x8e\x84\x49\x6a\xcc\x2a\x98\x29\x06\xe8\x74\x27\x0f\x82\xc7\xab\xeb\xaf\xe5\x43\xa0\x2d\x72\xfc\xf4\x16\x4b\x2b\x09\x03\x26\x1e\xc2\x6f\x5a\x35\x9a\x83\x06\xde\x1f\x59\x23\x25\x6d\x0d\x44\x22\x0f\xac\x1a\x7e\x8e\xdf\xce\xdd\x07\xb1\x6b\xd4\x0e\x9b\xe6\xb9\x79\x7f\xf4\x7c\xe4\xdb\x8a\x64\x0f\x33\x8e\x90\x5e\x0f\xd3\x54\xbd\x42\x1f\x23\xd4\xeb\xf3\x01\xb3\xda\x3b\x23\x2b\xb3\xe2\x08\x51\x71\x64\x1b\x19\x2e\x81\x85\xd5\xc3\x03\x9c\x13\x8a\xc3\x89\xcc\xeb\xc9\x82\xa1\xeb\x48\xf0\x6e\x7d\xa9\x41\xf7\xef\x89\x44\xbc\x2c\xc4\xc0\x25\x30\x83\x29\xab\xe1\xa8\xf1\x9f\x37\xef\xca\xd7\x41\x94\xa4\xa8\x4a\xe7\xb2\x9f\x74\x8f\x4c\x45\x5e\x95\xe4\x93\x73\x12\x14\xb9\x52\xeb\x2f\x09\xc7\xcf\x45\x8e\xac\x83\xd2\xdc\xea\xf2\x56\x75\x94\xc3\x01\xeb\x25\xcd\x44\xcf\xe5\x80\x47\xac\xee\xf8\x8c\x96\x56\x43\x59\xb0\x86\x83\x97\xf4\x7d\xf3\xe3\x69\xad\x44\xdb\x82\x1d\x35\x95\x17\x19\x22\x8a\xdc\x47\x8f\xf9\xf2\x09\x21\x66\x6f\x37\x7d\xc6\x38\xfe\x7f\x7b\xa5\x6e\x8e\xa0\x2f\x7d\x83\x05\x51\xd8\x37\x7d\xd2\x41\xc2\x1e\xbb\xd5\x6c\x83\x3b\x99\xbc\xe7\x23\x27\x13\x8f\xf7\xd5\xe5\x13\xad\x40\x62\xef\xe2\xe7\x8c\x37\x54\x77\xc9\x19\x3b\x87\xac\x85\x62\x8b\x31\x2f\x54\x1e\x16\x9d\x6b\x81\x45\x9e\xc3\x8e\x8b\x3a\xe4\x31\x36\xf6\x72\x2a\xc3\x53\x6f\x6f\xe1\x53\xd3\x88\x4b\xfa\xb7\x7f\x21\xf7\x47\x2f\x32\x0c\x43\xcc\xc6\x84\xb7\xa7\x32\x2d\x55\x43\x2a\x03\x92\x84\xdf\xb4\xd5\x62\x4f\xf5\x39\xc1\x7e\x89\x8c\x5d\xe7\xa7\x26\xb2\x76\x5d\x82\x9b\x06\x91\x73\x32\xe2\xde\x99\x5c\x93\xdf\x4a\x0e\x43\x34\xbe\x3e\xd4\x3d\x56\x3f\xc5\x75\x17\x87\x90\xfc\x25\xe3\x11\x8d\xf3\x89\x43\xe3\xd7\x1f\x6c\x71\x88\x05\xa3\xb6\xc1\x26\xc2\xc5\x9b\x1e\xb0\xa5\x35\xa3\x06\xbc\xec\x61\x88\x7b\xa5\x4b\x12\x30\x30\x96\x3c\x54\x3c\xdb\x88\x3d\x64\xbf\x37\x8f\x1e\xb7\x08\x78\x89\x49\xb8\x8d\x98\x2b\xf1\x34\x1f\x18\xe3\x3b\xfa\x7a\x9e\xae\x83\xe6\x57\xc1\x38\x0a\xad\xc3\x5a\xbd\xf0\xf5\x63\x3a\x84\xfd\x0b\x00\x00\xff\xff\xdb\x8e\x0c\x63\xab\x06\x00\x00")
+var _webUiTemplatesAlertsHtml = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x8c\x54\x4d\x8f\x9b\x30\x10\xbd\xef\xaf\xb0\xd0\x1e\x5a\xa9\x80\xd4\x63\x45\x90\x56\x7b\xe9\x61\x5b\x55\x9b\x74\xaf\x91\xb1\x27\x8b\xb7\x8e\x41\xb6\x93\x4d\xe4\xf2\xdf\x3b\xb6\x21\x4b\x08\x48\xbd\x24\x78\xe6\xcd\xf3\xf3\x7c\x39\xc7\x61\x27\x14\x90\xa4\x06\xca\x93\xae\xbb\x23\xa4\x90\x42\xfd\x21\xf6\xdc\xc2\x2a\xb1\x70\xb2\x39\x33\x26\x21\x1a\xe4\x2a\x31\xf6\x2c\xc1\xd4\x00\x36\x21\xb5\x86\xdd\x2a\x71\x8e\xb4\xd4\xd6\xbf\xf0\x20\x4e\xa4\xeb\x72\x63\xa9\x15\xcc\xc7\xe4\x54\x82\xb6\x26\xf3\xe1\xa5\xe7\x35\x4c\x8b\xd6\x12\xa3\xd9\x72\xdc\xdb\x25\xec\x0d\xa3\x8a\x3c\xc6\x94\x77\xce\x81\xe2\x28\xef\xee\x43\x31\x6b\x94\x05\x65\xbd\xe8\x82\x8b\x23\x61\x92\x1a\xb3\x0a\x66\x8a\x00\x9d\xee\xe4\x41\xf0\x78\x75\xfd\xb5\x7c\x08\xb4\x45\x8e\x9f\xde\x62\x69\x25\x61\x88\x89\x87\xf0\x9b\x56\x8d\xe6\xa0\x81\xf7\x47\xd6\x48\x49\x5b\x03\x91\xc8\x07\x56\x0d\x3f\xc7\x6f\xe7\xee\x83\xd8\x35\x6a\x87\x4d\xf3\xdc\xbc\x3f\x7a\x3e\xf2\x6d\x45\xb2\x87\x19\x47\x48\xaf\x0f\xd3\x54\xbd\x42\x8f\x11\xea\xf5\xf9\x80\x59\xed\x9d\x91\x95\x59\x71\x84\xa8\x38\xb2\x8d\x0c\x17\x60\x61\xf5\xf0\x00\xe7\x84\xe2\x70\x22\xf3\x7a\xb2\x60\xe8\x3a\x12\xbc\x5b\x5f\x6a\xd0\xfd\x7b\x22\x11\x2f\x0b\x31\x70\x09\xcc\x60\xca\x6a\x38\x6a\xfc\xe7\xcd\xbb\xf2\x75\x10\x25\x29\xaa\xd2\xb9\xec\x27\xdd\x23\x53\x91\x57\x25\xf9\xe4\x9c\x04\x45\xae\xd4\xfa\x4b\xc2\xf1\x73\x91\x23\xeb\xa0\x34\xb7\xba\xbc\x55\x1d\xe5\x70\xc0\x7a\x49\x33\xd1\x73\x39\xe0\x11\xab\x3b\x3e\xa3\xa5\xd5\x50\x16\xac\xe1\xe0\x25\x7d\xdf\xfc\x78\x5a\x2b\xd1\xb6\x60\x47\x4d\xe5\x45\x06\x44\x91\x7b\xf4\x98\x2f\x9f\x10\x62\xf6\x76\xd3\x67\x8c\xf1\xff\xdb\x2b\x75\x73\x04\x7d\xe9\x1b\x2c\x88\xc2\xbe\xe9\x93\x0e\x12\xf6\xd8\xad\x66\x1b\xdc\xc9\xe4\x3d\x1f\x39\x99\x78\xbc\xaf\x2e\x9f\x68\x05\x12\x7b\x17\x3f\x67\xbc\xa1\xba\x4b\xce\xd8\x39\x64\x2d\x14\x5b\xc4\xbc\x50\x79\x58\x74\xae\x05\x16\x79\x2e\x76\x5c\xd4\x21\x8f\xb1\xb1\x97\x53\x19\x9e\x7a\x7b\x0b\x9f\x9a\x46\x5c\xd2\xbf\xfd\x0b\xb9\x3f\x7a\x91\x61\x18\x62\x36\x26\xbc\x3d\x95\x69\xa9\x1a\x52\x19\x22\x49\xf8\x4d\x5b\x2d\xf6\x54\x9f\x13\xec\x97\xc8\xd8\x75\x7e\x6a\x22\x6b\xd7\x25\xb8\x69\x30\x72\x4e\x46\xdc\x3b\x93\x6b\xf2\x5b\xc9\x61\x88\xc6\xd7\x87\xba\xc7\xea\xa7\xb8\xee\xe2\x10\x92\xbf\x64\x3c\xa2\x71\x3e\x71\x68\xfc\xfa\x83\x2d\x0e\xb1\x60\xd4\x36\xd8\x44\xb8\x78\xd3\x03\xb6\xb4\x66\xd4\x80\x97\x3d\x0c\x71\xaf\x74\x49\x02\x02\xfb\x65\x61\xb3\x8d\xd8\x43\xf6\x7b\xf3\xe8\x83\x16\xd1\x2f\x31\x03\xb7\x88\xb9\xfa\x4e\x93\x81\x18\xdf\xce\xd7\xc3\x74\x0d\x9a\xdf\x03\x63\x14\x5a\x87\x9d\x7a\xe1\xeb\x67\x74\x80\xfd\x0b\x00\x00\xff\xff\xb3\xf6\xaa\xca\xa8\x06\x00\x00")
 
 func webUiTemplatesAlertsHtmlBytes() ([]byte, error) {
 	return bindataRead(
@@ -134,7 +134,7 @@ func webUiTemplatesAlertsHtml() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1707, mode: os.FileMode(420), modTime: time.Unix(1450348618, 0)}
+	info := bindataFileInfo{name: "web/ui/templates/alerts.html", size: 1704, mode: os.FileMode(420), modTime: time.Unix(1450348695, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }

web/ui/templates/alerts.html

@@ -36,7 +36,7 @@
                 {{end}}
               </td>
               <td><span class="alert alert-{{ .State | alertStateToClass }} state_indicator text-uppercase">{{.State}}</span></td>
-              <td>{{.ActiveSince.Time.UTC}}</td>
+              <td>{{.ActiveAt.Time.UTC}}</td>
               <td>{{.Value}}</td>
             </tr>
             {{end}}