From bbbe7474976e33c5471f7ecdabd4cf2a44c25ee2 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 26 Jun 2024 20:34:15 +0100 Subject: [PATCH] Set a minimum threshold for error rate alerts This avoids very small numbers of errors on idle machines triggering an errir rate alert. --- cookbooks/prometheus/templates/default/alert_rules.yml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb index dfb83891d..b78bbaddd 100644 --- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb +++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb @@ -798,7 +798,7 @@ groups: - name: web rules: - alert: web error rate - expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 + expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 and sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) > 0.01 for: 5m labels: alertgroup: web -- 2.39.5