From 94fc82e4183630050523535ff1683ab090349d53 Mon Sep 17 00:00:00 2001
From: Vitaly Zhuravlev <v-zhuravlev@users.noreply.github.com>
Date: Tue, 28 Mar 2023 05:26:30 +0800
Subject: [PATCH] Add NodeDiskIOSaturation alert

Signed-off-by: Vitaly Zhuravlev <v-zhuravlev@users.noreply.github.com>
---
 docs/node-mixin/alerts/alerts.libsonnet | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet
index 6c0ef5b4..bcd955f1 100644
--- a/docs/node-mixin/alerts/alerts.libsonnet
+++ b/docs/node-mixin/alerts/alerts.libsonnet
@@ -333,10 +333,27 @@
               severity: 'warning',
             },
             annotations: {
-              summary: 'Host is running out of memory',
+              summary: 'Host is running out of memory.',
               description: 'Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.',
             },
           },
+          {
+            alert: 'NodeDiskIOSaturation',
+            expr: |||
+              rate(node_disk_io_time_weighted_seconds_total{%(nodeExporterSelector)s}[5m]) > 10
+            ||| % $._config,
+            'for': '30m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'Disk IO queue is high.',
+              description: |||
+                Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
+                This symptom might indicate disk saturation.,
+              |||,
+            },
+          },
           {
             alert: 'NodeSystemdServiceFailed',
             expr: |||