Fix alerting for failed chef runs

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 1083f74f77b547722d48fcaad22d60021c60a798..7a535337264c8a90d42bd00653ecc5ff64ddb12a 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -623,7 +623,7 @@ groups:
         labels:
           alertgroup: "{{ $labels.instance }}"
       - alert: systemd failed chef client service
         labels:
           alertgroup: "{{ $labels.instance }}"
       - alert: systemd failed chef client service

-        expr: node_systemd_unit_state{state="inactive",name="chef-client.service"}[6h] == 0
+        expr: sum_over_time(node_systemd_unit_state{state="inactive",name="chef-client.service"}[6h]) == 0

         for: 0m
         labels:
           alertgroup: "{{ $labels.instance }}"
         for: 0m
         labels:
           alertgroup: "{{ $labels.instance }}"