From: Tom Hughes <tom@compton.nu>
Date: Wed, 22 Dec 2021 19:39:44 +0000 (+0000)
Subject: Add alert for high error rates on fastly
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/84625fc5630ce46c37f93be62aa61c67cbfc4ef0?hp=9f8c68c388d4ce13634121e808ca06cca75f9501

Add alert for high error rates on fastly
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 562d31046..92c38d5c5 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -40,6 +40,15 @@ groups:
           alertgroup: database
         annotations:
           delay: "{{ $value | humanizeDuration }}"
+  - name: fastly
+    rules:
+      - alert: error rate
+        expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005
+        for: 15m
+        labels:
+          alertgroup: fastly
+        annotations:
+          error_rate: "{{ $value | humanizePercentage }}"
   - name: filesystem
     rules:
       - alert: readonly filesystem