From: Tom Hughes <tom@compton.nu>
Date: Sun, 14 Nov 2021 17:48:05 +0000 (+0000)
Subject: Add alerts for filesystems low on space
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/b99e4a5a8bf96c91fa1af0991d249a3c4bbabb46?ds=sidebyside

Add alerts for filesystems low on space
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 4511e1eac..1b3afcba5 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -31,6 +31,26 @@ groups:
           alertgroup: database
         annotations:
           delay: "{{ $value | humanizeDuration }}"
+  - name: filesystem
+    rules:
+      - alert: filesystem low on space
+        expr: node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.05
+        for: 5m
+        labels:
+          alertgroup: "{{ $labels.instance }}"
+        annotations:
+          percentage_free: "{{ $value | humanizePercentage }}"
+          free_bytes: "{{ with printf \"node_filesystem_avail_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
+          total_total: "{{ with printf \"node_filesystem_size_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
+      - alert: filesystem low on inodes
+        expr: node_filesystem_files_free / node_filesystem_files < 0.1
+        for: 5m
+        labels:
+          alertgroup: "{{ $labels.instance }}"
+        annotations:
+          percentage_free: "{{ $value | humanizePercentage }}"
+          free_inodes: "{{ with printf \"node_filesystem_files_free{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
+          total_inodes: "{{ with printf \"node_filesystem_files{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
   - name: hwmon
     rules:
       - alert: hwmon fan alarm