From ae501e32e6b6b901122e511470c2f838cd1b9e0a Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Mon, 10 Jul 2023 14:20:15 +0100 Subject: [PATCH] Improve fastly alerts to make them more useful --- .../templates/default/alert_rules.yml.erb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb index 396de8de4..899c765e4 100644 --- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb +++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb @@ -160,13 +160,23 @@ groups: alertgroup: fastly annotations: error_rate: "{{ $value | humanizePercentage }}" - - alert: fastly healthcheck failing - expr: count(fastly_healthcheck_status == 0) by (service) > 0 + - alert: fastly frontend healthcheck warning + expr: count(fastly_healthcheck_status == 0) by (service, datacenter) > 2 for: 15m labels: alertgroup: fastly - - alert: multiple fastly healthchecks failing - expr: count(fastly_healthcheck_status == 0) by (service) > 4 + - alert: fastly frontend healthcheck critical + expr: count(fastly_healthcheck_status == 0) by (service, datacenter) == count(fastly_healthcheck_status) by (service, datacenter) + for: 5m + labels: + alertgroup: fastly + - alert: fastly backend healthcheck warning + expr: count(fastly_healthcheck_status == 0) by (service, backend) > 10 + for: 15m + labels: + alertgroup: fastly + - alert: fastly backend healthcheck critical + expr: count(fastly_healthcheck_status == 0) by (service, backend) == count(fastly_healthcheck_status) by (service, backend) for: 5m labels: alertgroup: fastly -- 2.39.5