Add alert for job processing rate

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 92c38d5c5d1f35e54db24742a8a74747aa7b21b4..be6a89b4340a7d5764b3874ddb6ba0dcbaf74671 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -410,3 +410,10 @@ groups:
           alertgroup: web
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
           alertgroup: web
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"

+      - alert: job processing rate
+        expr: rate(pg_stat_user_tables_n_tup_del{datname="openstreetmap",relname="delayed_jobs"}[5m]) / rate(pg_stat_user_tables_n_tup_ins{datname="openstreetmap",relname="delayed_jobs"}[5m]) < 0.9 and ignoring(job, name, datname, relname, schemaname, server) chef_role{name="db-master"} == 1
+        for: 5m
+        labels:
+          alertgroup: web
+        annotations:
+          job_processing_rate: "{{ $value | humanizePercentage }}"