From: Tom Hughes <tom@compton.nu>
Date: Mon, 21 Nov 2022 21:00:45 +0000 (+0000)
Subject: Make fastly healthcheck alerts operate on a per-service basis
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/9e2201253ded5552159a233d98c5003582a0c233?ds=inline

Make fastly healthcheck alerts operate on a per-service basis
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 9522dbf6b..8c1aeee48 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -145,12 +145,12 @@ groups:
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
       - alert: fastly healthcheck failing
-        expr: count(fastly_healthcheck_status == 0) > 0
+        expr: count(fastly_healthcheck_status == 0) by (service) > 0
         for: 15m
         labels:
           alertgroup: fastly
       - alert: multiple fastly healthchecks failing
-        expr: count(fastly_healthcheck_status == 0) > 4
+        expr: count(fastly_healthcheck_status == 0) by (service) > 4
         for: 5m
         labels:
           alertgroup: fastly