Add alert for WAL remote write falling behind.

Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>

documentation/prometheus-mixin/alerts.libsonnet

@@ -156,6 +156,22 @@
               message: 'Prometheus failed to send {{ printf "%.1f" $value }}% samples',
             },
           },
+          {
+            alert: 'PrometheusRemoteWriteBehind',
+            expr: |||
+              prometheus_remote_storage_highest_timestamp_in{%(prometheusSelector)s}
+                - on(job, instance) group_right
+              prometheus_remote_storage_queue_highest_sent_timestamp{%(prometheusSelector)s}
+                > 60
+            ||| % $._config,
+            'for': '15m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              message: 'Prometheus remote write is {{ printf "%.1f" $value }}s behind.',
+            },
+          },
           {
             alert: 'PrometheusRuleFailures',
             'for': '15m',