Add alert for failing discourse jobs

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 848123602032a5cca93197285fa244009bb9022c..239731607087fb5c1dc9d6854797c0460be9e0ba 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -105,6 +105,15 @@ groups:
           alertgroup: database
         annotations:
           delay: "{{ $value | humanizeDuration }}"
           alertgroup: database
         annotations:
           delay: "{{ $value | humanizeDuration }}"

+  - name: discourse
+    rules:
+      - alert: discourse job failure rate
+        expr: rate(discourse_job_failures[5m]) > 0
+        for: 5m
+        labels:
+          alertgroup: discourse
+        annotations:
+          failure_rate: "{{ $value }} jobs/s"

   - name: dublin
     rules:
       - alert: pdu current draw
   - name: dublin
     rules:
       - alert: pdu current draw