From: Tom Hughes <tom@compton.nu>
Date: Wed, 20 Jan 2021 19:03:36 +0000 (+0000)
Subject: Configure prometheus alert manager
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/ac279eeeae75039bf84ffffdbb849b37bf554a75

Configure prometheus alert manager
---

diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb
index 571714c50..9acc28671 100644
--- a/cookbooks/prometheus/recipes/server.rb
+++ b/cookbooks/prometheus/recipes/server.rb
@@ -32,7 +32,10 @@ prometheus_exporter "fastly" do
   environment "FASTLY_API_TOKEN" => tokens["fastly"]
 end
 
-package "prometheus"
+package %w[
+  prometheus
+  prometheus-alertmanager
+]
 
 promscale_version = "0.1.4"
 
@@ -179,6 +182,18 @@ service "prometheus" do
   subscribes :reload, "template[/etc/prometheus/prometheus.yml]"
 end
 
+template "/etc/prometheus/alertmanager.yml" do
+  source "alertmanager.yml.erb"
+  owner "root"
+  group "root"
+  mode "644"
+end
+
+service "prometheus-alertmanager" do
+  action [:enable, :start]
+  subscribes :reload, "template[/etc/prometheus/alertmanager.yml]"
+end
+
 package "grafana-enterprise"
 
 template "/etc/grafana/grafana.ini" do
diff --git a/cookbooks/prometheus/templates/default/alertmanager.yml.erb b/cookbooks/prometheus/templates/default/alertmanager.yml.erb
new file mode 100644
index 000000000..4df70fd94
--- /dev/null
+++ b/cookbooks/prometheus/templates/default/alertmanager.yml.erb
@@ -0,0 +1,23 @@
+# DO NOT EDIT - This file is being maintained by Chef
+
+global:
+  smtp_smarthost: localhost:25
+  smtp_from: prometheus@openstreetmap.org
+
+templates:
+  - /etc/prometheus/alertmanager_templates/*.tmpl
+
+route:
+  group_by:
+    - service
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 3h
+  receiver: admins-email
+
+inhibit_rules:
+
+receivers:
+  - name: admins-email
+    email_configs:
+      - to: admins@openstreetmap.org
diff --git a/roles/mail.rb b/roles/mail.rb
index 660bb9f2f..5783e1bda 100644
--- a/roles/mail.rb
+++ b/roles/mail.rb
@@ -58,6 +58,7 @@ default_attributes(
       "rails" => "root",
       "trac" => "root",
       "munin" => "root",
+      "prometheus" => "root",
       "www-data" => "root",
       "osmbackup" => "root",
       "noreply" => "/dev/null",
diff --git a/test/integration/prometheus-server/serverspec/alertmanager_spec.rb b/test/integration/prometheus-server/serverspec/alertmanager_spec.rb
new file mode 100644
index 000000000..cdad78319
--- /dev/null
+++ b/test/integration/prometheus-server/serverspec/alertmanager_spec.rb
@@ -0,0 +1,17 @@
+require "serverspec"
+
+# Required by serverspec
+set :backend, :exec
+
+describe package("prometheus-alertmanager") do
+  it { should be_installed }
+end
+
+describe service("prometheus-alertmanager") do
+  it { should be_enabled }
+  it { should be_running }
+end
+
+describe port(9093) do
+  it { should be_listening.with("tcp6") }
+end