From: Tom Hughes <tom@compton.nu>
Date: Thu, 24 Feb 2022 22:52:42 +0000 (+0000)
Subject: Add alert for job processing rate
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/aad368fcb42c6955ccadbabe1f22aca5d31730dd?hp=a7cd6577d87257b9fd990f97803b77ce12213b7f

Add alert for job processing rate
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 92c38d5c5..be6a89b43 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -410,3 +410,10 @@ groups:
           alertgroup: web
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
+      - alert: job processing rate
+        expr: rate(pg_stat_user_tables_n_tup_del{datname="openstreetmap",relname="delayed_jobs"}[5m]) / rate(pg_stat_user_tables_n_tup_ins{datname="openstreetmap",relname="delayed_jobs"}[5m]) < 0.9 and ignoring(job, name, datname, relname, schemaname, server) chef_role{name="db-master"} == 1
+        for: 5m
+        labels:
+          alertgroup: web
+        annotations:
+          job_processing_rate: "{{ $value | humanizePercentage }}"