From: Tom Hughes <tom@compton.nu>
Date: Mon, 27 Feb 2023 22:33:53 +0000 (+0000)
Subject: Fix alerting for failed chef runs
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/a052eb992b55917d502d1301f2c1b7d6f5fb81c9

Fix alerting for failed chef runs
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 01d8ae00f..1083f74f7 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -623,8 +623,8 @@ groups:
         labels:
           alertgroup: "{{ $labels.instance }}"
       - alert: systemd failed chef client service
-        expr: node_systemd_unit_state{state="failed",name="chef-client.service"} == 1
-        for: 6h
+        expr: node_systemd_unit_state{state="inactive",name="chef-client.service"}[6h] == 0
+        for: 0m
         labels:
           alertgroup: "{{ $labels.instance }}"
   - name: tile