From 5d713dc40aee7ccf357a1b38d1dd6e921ce33952 Mon Sep 17 00:00:00 2001
From: Justin Lei <lei.justin@gmail.com>
Date: Thu, 21 Nov 2024 15:00:28 -0800
Subject: [PATCH] Add differentiating label for alerts that use
 nodeCriticalSeverity

This affects the following alerts:

- NodeFilesystemAlmostOutOfFiles
- NodeFilesystemSpaceFillingUp
- NodeFilesystemAlmostOutOfSpace
- NodeFilesystemFilesFillingUp

Each alert is actually two alerting rules, one for the warning level,
and one for the critical level. However, the critical one can be set to
emit as a warning by using $._config.nodeCriticalSeverity. When this is
true, both alerting rules will generate alerts with an identical set of
labels, which is an error when both alerts fire.

We add an extra label to these alerting rules in order to differentiate
between the two levels when $._config.nodeCriticalSeverity is set to
`warning`.

Signed-off-by: Justin Lei <lei.justin@gmail.com>
---
 docs/node-mixin/alerts/alerts.libsonnet | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet
index c3464809..dd2fb44e 100644
--- a/docs/node-mixin/alerts/alerts.libsonnet
+++ b/docs/node-mixin/alerts/alerts.libsonnet
@@ -18,6 +18,7 @@
             'for': '1h',
             labels: {
               severity: 'warning',
+              window: '24h',
             },
             annotations: {
               summary: 'Filesystem is predicted to run out of space within the next 24 hours.',
@@ -38,6 +39,7 @@
             'for': '1h',
             labels: {
               severity: '%(nodeCriticalSeverity)s' % $._config,
+              window: '4h',
             },
             annotations: {
               summary: 'Filesystem is predicted to run out of space within the next 4 hours.',
@@ -56,6 +58,7 @@
             'for': '30m',
             labels: {
               severity: 'warning',
+              threshold: '%(fsSpaceAvailableWarningThreshold)d%%' % $._config,
             },
             annotations: {
               summary: 'Filesystem has less than %(fsSpaceAvailableWarningThreshold)d%% space left.' % $._config,
@@ -74,6 +77,7 @@
             'for': '30m',
             labels: {
               severity: '%(nodeCriticalSeverity)s' % $._config,
+              threshold: '%(fsSpaceAvailableCriticalThreshold)d%%' % $._config,
             },
             annotations: {
               summary: 'Filesystem has less than %(fsSpaceAvailableCriticalThreshold)d%% space left.' % $._config,
@@ -94,6 +98,7 @@
             'for': '1h',
             labels: {
               severity: 'warning',
+              window: '24h',
             },
             annotations: {
               summary: 'Filesystem is predicted to run out of inodes within the next 24 hours.',
@@ -114,6 +119,7 @@
             'for': '1h',
             labels: {
               severity: '%(nodeCriticalSeverity)s' % $._config,
+              window: '4h',
             },
             annotations: {
               summary: 'Filesystem is predicted to run out of inodes within the next 4 hours.',
@@ -132,6 +138,7 @@
             'for': '1h',
             labels: {
               severity: 'warning',
+              threshold: '5%',
             },
             annotations: {
               summary: 'Filesystem has less than 5% inodes left.',
@@ -150,6 +157,7 @@
             'for': '1h',
             labels: {
               severity: '%(nodeCriticalSeverity)s' % $._config,
+              threshold: '3%',
             },
             annotations: {
               summary: 'Filesystem has less than 3% inodes left.',