X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/cba7532d66e48bf49632eaf22d4ba9407b8182af..ed524a3afc8f43c16510de915662632e3b186a88:/cookbooks/prometheus/templates/default/alert_rules.yml.erb

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index a52901233..28a5b3113 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -1,13 +1,6 @@
 # DO NOT EDIT - This file is being maintained by Chef
 
 groups:
-  - name: alertmanager
-    rules:
-      - alert: prometheus target missing
-        expr: up == 0
-        for: 10m
-        labels:
-          alertgroup: "prometheus"
   - name: amsterdam
     rules:
       - alert: pdu current draw
@@ -114,13 +107,23 @@ groups:
           delay: "{{ $value | humanizeDuration }}"
   - name: fastly
     rules:
-      - alert: error rate
+      - alert: fastly error rate
         expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005
         for: 15m
         labels:
           alertgroup: fastly
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
+      - alert: fastly healthcheck failing
+        expr: count(fastly_healthcheck_status == 0) > 0
+        for: 15m
+        labels:
+          alertgroup: fastly
+      - alert: fastly healthcheck failing
+        expr: count(fastly_healthcheck_status == 0) > 4
+        for: 5m
+        labels:
+          alertgroup: fastly
   - name: filesystem
     rules:
       - alert: readonly filesystem
@@ -417,6 +420,30 @@ groups:
           alertgroup: "{{ $labels.instance }}"
         annotations:
           queries: "{{ $value }}"
+  - name: prometheus
+    rules:
+      - alert: prometheus configuration error
+        expr: prometheus_config_last_reload_successful == 0
+        for: 10m
+        labels:
+          alertgroup: "prometheus"
+      - alert: prometheus target missing
+        expr: up == 0
+        for: 10m
+        labels:
+          alertgroup: "prometheus"
+  - name: raid
+    rules:
+      - alert: raid array degraded
+        expr: ohai_array_info{status="degraded"} > 0
+        for: 5m
+        labels:
+          alertgroup: "{{ $labels.instance }}"
+      - alert: raid disk failed
+        expr: ohai_disk_info{status="failed"} > 0
+        for: 5m
+        labels:
+          alertgroup: "{{ $labels.instance }}"
   - name: smart
     rules:
       - alert: smart failure
@@ -425,7 +452,7 @@ groups:
         labels:
           alertgroup: "{{ $labels.instance }}"
       - alert: smart ssd wearout approaching
-        expr: smart_percentage_used >= 90
+        expr: smart_percentage_used >= 80
         for: 60m
         labels:
           alertgroup: "{{ $labels.instance }}"