From 55ffe57cbc00d278cf788ad98512c281ea891cb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Blot?= <nerzhul@users.noreply.github.com>
Date: Fri, 30 Apr 2021 12:01:51 +0200
Subject: [PATCH] feat(rules): add NodeFileDescriptorLimit kernel exhaustion
alert
Add a new alert when fs.file-nr is close to fs.file-max
Signed-off-by: Loic Blot <loic.blot@unix-experience.fr>
---
docs/node-mixin/alerts/alerts.libsonnet | 32 +++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet
index a19c8dc9..f39ef52c 100644
--- a/docs/node-mixin/alerts/alerts.libsonnet
+++ b/docs/node-mixin/alerts/alerts.libsonnet
@@ -277,6 +277,38 @@
description: "At least one device in RAID array on {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.",
},
},
+ {
+ alert: 'NodeFileDescriptorLimit',
+ expr: |||
+ (
+ node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
+ )
+ ||| % $._config,
+ 'for': '15m',
+ labels: {
+ severity: 'warning',
+ },
+ annotations: {
+ summary: 'Kernel is predicted to exhaust file descriptors limit soon.',
+ description: 'File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.',
+ },
+ },
+ {
+ alert: 'NodeFileDescriptorLimit',
+ expr: |||
+ (
+ node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
+ )
+ ||| % $._config,
+ 'for': '15m',
+ labels: {
+ severity: 'critical',
+ },
+ annotations: {
+ summary: 'Kernel is predicted to exhaust file descriptors limit soon.',
+ description: 'File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.',
+ },
+ },
],
},
],