From: Tom Hughes <tom@compton.nu>
Date: Sat, 10 Dec 2022 12:14:03 +0000 (+0000)
Subject: Only alert for readonly filesystems which were recently writable
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/694ab226ea187543b7987cc396853f2c2515df7c

Only alert for readonly filesystems which were recently writable
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 149ff86bc..16496c12d 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -157,7 +157,7 @@ groups:
   - name: filesystem
     rules:
       - alert: readonly filesystem
-        expr: node_filesystem_readonly == 1
+        expr: node_filesystem_readonly > min_over_time(node_filesystem_readonly[7d])
         for: 0m
         labels:
           alertgroup: "{{ $labels.instance }}"