Add an alert for chef not running for an extended time

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 2aa46d1f0d09d77dbf110c9620720af81f59690b..1f0ae128e4d967808421316f8de279376d1087cb 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -22,6 +22,15 @@ groups:
           alertgroup: "{{ $labels.instance }}"
         annotations:
           busy_workers: "{{ $value | humanizePercentage }}"
+  - name: chef
+    rules:
+      - alert: chef client not running
+        expr: time() - node_systemd_timer_last_trigger_seconds{name="chef-client.timer"} > 3600
+        for: 12h
+        labels:
+          alertgroup: "{{ $labels.instance }}"
+        annotations:
+          down_time: "{{ $value | humanizeDuration }}"
   - name: cpu
     rules:
       - alert: cpu pressure