diff --git a/notification/instrumentation.go b/notification/instrumentation.go new file mode 100644 index 000000000..1556263ff --- /dev/null +++ b/notification/instrumentation.go @@ -0,0 +1,54 @@ +// Copyright 2013 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package notification + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + result = "result" + success = "success" + failure = "failure" + dropped = "dropped" + + facet = "facet" + occupancy = "occupancy" + capacity = "capacity" +) + +var ( + notificationsCount = prometheus.NewCounter() + notificationLatency = prometheus.NewDefaultHistogram() + notificationsQueueSize = prometheus.NewGauge() +) + +func recordOutcome(duration time.Duration, err error) { + labels := map[string]string{result: success} + if err != nil { + labels[result] = failure + } + + notificationsCount.Increment(labels) + ms := float64(duration / time.Millisecond) + notificationLatency.Add(labels, ms) +} + +func init() { + prometheus.Register("prometheus_notifications_total", "Total number of processed alert notifications.", prometheus.NilLabels, notificationsCount) + prometheus.Register("prometheus_notifications_latency_ms", "Latency quantiles for sending alert notifications in milliseconds.", prometheus.NilLabels, notificationLatency) + prometheus.Register("prometheus_notifications_queue_size_total", "The size and capacity of the alert notification queue.", prometheus.NilLabels, notificationsQueueSize) +} diff --git a/notification/notification.go b/notification/notification.go index 6df5c6de3..3f8873fe6 100644 --- a/notification/notification.go +++ b/notification/notification.go @@ -95,13 +95,34 @@ func (n *NotificationHandler) sendNotifications(reqs rules.NotificationReqs) err return nil } -// Continusouly dispatch notifications. +// Report notification queue occupancy and capacity. +func (n *NotificationHandler) reportQueues() { + notificationsQueueSize.Set(map[string]string{facet: occupancy}, float64(len(n.pendingNotifications))) + notificationsQueueSize.Set(map[string]string{facet: capacity}, float64(cap(n.pendingNotifications))) +} + +// Continuously dispatch notifications. func (n *NotificationHandler) Run() { + queueReportTicker := time.NewTicker(time.Second) + go func() { + for _ = range queueReportTicker.C { + n.reportQueues() + } + }() + defer queueReportTicker.Stop() + for reqs := range n.pendingNotifications { if n.alertmanagerUrl == "" { log.Println("No alert manager configured, not dispatching notification") + notificationsCount.Increment(map[string]string{result: dropped}) + continue } - if err := n.sendNotifications(reqs); err != nil { + + begin := time.Now() + err := n.sendNotifications(reqs) + recordOutcome(time.Since(begin), err) + + if err != nil { log.Println("Error sending notification:", err) } } diff --git a/storage/metric/instrumentation.go b/storage/metric/instrumentation.go index 71e95238d..dbef506ba 100644 --- a/storage/metric/instrumentation.go +++ b/storage/metric/instrumentation.go @@ -14,8 +14,9 @@ package metric import ( - "github.com/prometheus/client_golang/prometheus" "time" + + "github.com/prometheus/client_golang/prometheus" ) const (