Only alert for failed chef-client services if they persist

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index be6a89b4340a7d5764b3874ddb6ba0dcbaf74671..2aa46d1f0d09d77dbf110c9620720af81f59690b 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -367,10 +367,15 @@ groups:
   - name: systemd
     rules:
       - alert: systemd failed service
-        expr: node_systemd_unit_state{state="failed"} == 1
+        expr: node_systemd_unit_state{state="failed",name!="chef-client.service"} == 1
         for: 5m
         labels:
           alertgroup: "{{ $labels.instance }}"
+      - alert: systemd failed service
+        expr: node_systemd_unit_state{state="failed",name="chef-client.service"} == 1
+        for: 6h
+        labels:
+          alertgroup: "{{ $labels.instance }}"
   - name: tile
     rules:
       - alert: renderd replication delay