Add alert for failed services

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 1b3afcba59276b20205a54007162da3899da0956..b9ffa9f44cff332878a4e0352cdbb85f296dfe06 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -262,6 +262,13 @@ groups:
         for: 0m
         labels:
           alertgroup: ssl
         for: 0m
         labels:
           alertgroup: ssl

+  - name: systemd
+    rules:
+      - alert: systemd failed service
+        expr: node_systemd_unit_state{state="failed"} == 1
+        for: 5m
+        labels:
+          alertgroup: "{{ $labels.instance }}"

   - name: tile
     rules:
       - alert: renderd replication delay
   - name: tile
     rules:
       - alert: renderd replication delay