From 877320784b8aa3b30835cd51eb95518886f7d697 Mon Sep 17 00:00:00 2001 From: fpetkovski Date: Wed, 23 Mar 2022 14:44:42 +0100 Subject: [PATCH] Add alert in mixin for exceeded sample limit This commit adds an alert in the prometheus mixin which triggers when Prometheus has failed scrapes that have exceeded the configured sample_limit for that job. Signed-off-by: fpetkovski --- documentation/prometheus-mixin/alerts.libsonnet | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/documentation/prometheus-mixin/alerts.libsonnet b/documentation/prometheus-mixin/alerts.libsonnet index 61328dc0a..bc77df55c 100644 --- a/documentation/prometheus-mixin/alerts.libsonnet +++ b/documentation/prometheus-mixin/alerts.libsonnet @@ -289,6 +289,20 @@ description: 'Prometheus %(prometheusName)s has dropped {{ printf "%%.0f" $value }} targets because some targets exceeded the configured body_size_limit.' % $._config, }, }, + { + alert: 'PrometheusScrapeSampleLimitHit', + expr: ||| + increase(prometheus_target_scrapes_exceeded_sample_limit_total{%(prometheusSelector)s}[5m]) > 0 + ||| % $._config, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Prometheus has failed scrapes that have exceeded the configured sample limit.', + description: 'Prometheus %(prometheusName)s has failed {{ printf "%%.0f" $value }} scrapes because some targets exceeded the configured sample_limit.' % $._config, + }, + }, { alert: 'PrometheusTargetSyncFailure', expr: |||