From 5710ddf24fd9a15710b391400aae4e4cc426e97a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 26 Sep 2024 15:32:18 +0100 Subject: [PATCH] [ENHANCEMENT] Alerts: remove metrics for removed Alertmanagers (#13909) * [ENHANCEMENT] Alerts: remove metrics for removed Alertmanagers So they don't continue to report stale values. Signed-off-by: Bryan Boreham --- notifier/notifier.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/notifier/notifier.go b/notifier/notifier.go index 218e4cb8c..5374e73d6 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -770,6 +770,7 @@ func (s *alertmanagerSet) sync(tgs []*targetgroup.Group) { s.mtx.Lock() defer s.mtx.Unlock() + previousAms := s.ams // Set new Alertmanagers and deduplicate them along their unique URL. s.ams = []alertmanager{} s.droppedAms = []alertmanager{} @@ -789,6 +790,17 @@ func (s *alertmanagerSet) sync(tgs []*targetgroup.Group) { seen[us] = struct{}{} s.ams = append(s.ams, am) } + // Now remove counters for any removed Alertmanagers. + for _, am := range previousAms { + us := am.url().String() + if _, ok := seen[us]; ok { + continue + } + s.metrics.latency.DeleteLabelValues(us) + s.metrics.sent.DeleteLabelValues(us) + s.metrics.errors.DeleteLabelValues(us) + seen[us] = struct{}{} + } } func postPath(pre string, v config.AlertmanagerAPIVersion) string {