From: Tom Hughes <tom@compton.nu>
Date: Sat, 21 Oct 2023 11:51:32 +0000 (+0100)
Subject: Backup prometheus data to S3
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/1dbda5d6ff8491bc544e44659f369388b77d8645

Backup prometheus data to S3
---

diff --git a/cookbooks/prometheus/metadata.rb b/cookbooks/prometheus/metadata.rb
index f79ad0bcb..6148979b9 100644
--- a/cookbooks/prometheus/metadata.rb
+++ b/cookbooks/prometheus/metadata.rb
@@ -8,6 +8,7 @@ version           "1.0.0"
 supports          "ubuntu"
 depends           "apache"
 depends           "apt"
+depends           "awscli"
 depends           "git"
 depends           "hardware"
 depends           "networking"
diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb
index 3014cbf99..d16d47fda 100644
--- a/cookbooks/prometheus/recipes/server.rb
+++ b/cookbooks/prometheus/recipes/server.rb
@@ -19,6 +19,7 @@
 
 include_recipe "apache"
 include_recipe "apt::grafana"
+include_recipe "awscli"
 include_recipe "networking"
 
 passwords = data_bag_item("prometheus", "passwords")
@@ -377,3 +378,42 @@ template "/etc/cron.daily/prometheus-backup" do
   group "root"
   mode "750"
 end
+
+package %w[
+  curl
+  jq
+]
+
+directory "/var/lib/prometheus/.aws" do
+  user "prometheus"
+  group "prometheus"
+  mode "755"
+end
+
+template "/var/lib/prometheus/.aws/credentials" do
+  source "aws-credentials.erb"
+  user "prometheus"
+  group "prometheus"
+  mode "600"
+  variables :passwords => passwords
+end
+
+template "/usr/local/bin/prometheus-backup-data" do
+  source "backup-data.erb"
+  owner "root"
+  group "root"
+  mode "755"
+end
+
+systemd_service "prometheus-backup-data" do
+  description "Backup prometheus data to S3"
+  user "prometheus"
+  exec_start "/usr/local/bin/prometheus-backup-data"
+  read_write_paths "/var/lib/prometheus/metrics2/snapshots"
+  sandbox :enable_network => true
+end
+
+systemd_timer "prometheus-backup-data" do
+  description "Backup prometheus data to S3"
+  on_calendar "03:11"
+end
diff --git a/cookbooks/prometheus/templates/default/aws-credentials.erb b/cookbooks/prometheus/templates/default/aws-credentials.erb
new file mode 100644
index 000000000..9ee21b85d
--- /dev/null
+++ b/cookbooks/prometheus/templates/default/aws-credentials.erb
@@ -0,0 +1,7 @@
+[osm-prometheus-data]
+aws_access_key_id = AKIASQUXHPE7KAYP364J
+aws_secret_access_key = <%= @passwords["aws_prometheus_data"] %>
+
+[osm-prometheus-data-upload]
+role_arn=arn:aws:iam::173189593406:role/osm-prometheus-data-upload-role
+source_profile=osm-prometheus-data
diff --git a/cookbooks/prometheus/templates/default/backup-data.erb b/cookbooks/prometheus/templates/default/backup-data.erb
new file mode 100644
index 000000000..e6dabd4c6
--- /dev/null
+++ b/cookbooks/prometheus/templates/default/backup-data.erb
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+SNAPSHOT=$(curl -XPOST http://localhost:9090/prometheus/api/v1/admin/tsdb/snapshot | jq -r .data.name)
+
+/opt/awscli/v2/current/bin/aws --profile osm-prometheus-data-upload s3 sync --storage-class=INTELLIGENT_TIERING --no-progress /var/lib/prometheus/metrics2/snapshots/${SNAPSHOT} s3://openstreetmap-prometheus-data
+
+rm -rf /var/lib/prometheus/metrics2/snapshots/${SNAPSHOT}