Add an alert for failing healthchecks on the CDN

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index a52901233210ae425d632652578d92f4badbdf75..c108451abe8e362b103e58f24e8a14aef9351613 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -114,13 +114,18 @@ groups:
           delay: "{{ $value | humanizeDuration }}"
   - name: fastly
     rules:
           delay: "{{ $value | humanizeDuration }}"
   - name: fastly
     rules:

-      - alert: error rate
+      - alert: fastly error rate

         expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005
         for: 15m
         labels:
           alertgroup: fastly
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
         expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005
         for: 15m
         labels:
           alertgroup: fastly
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"

+      - alert: fastly healthcheck failing
+        expr: fastly_healthcheck_status = 0
+        for: 5m
+        labels:
+          alertgroup: fastly

   - name: filesystem
     rules:
       - alert: readonly filesystem
   - name: filesystem
     rules:
       - alert: readonly filesystem