From b68187fc95272d0d940e2fa6eb9ef8c4efd25048 Mon Sep 17 00:00:00 2001
From: Tom Hughes <tom@compton.nu>
Date: Sun, 24 Sep 2023 10:41:09 +0100
Subject: [PATCH] Copy changeset replication diffs to S3

---
 .../default/replication-bin/replicate-changesets     | 12 ++++++++++++
 cookbooks/planet/recipes/replication.rb              |  1 +
 .../planet/templates/default/changesets.conf.erb     |  1 +
 3 files changed, 14 insertions(+)

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/files/default/replication-bin/replicate-changesets
index 8731c0d20..87334f21a 100755
--- a/cookbooks/planet/files/default/replication-bin/replicate-changesets
+++ b/cookbooks/planet/files/default/replication-bin/replicate-changesets
@@ -207,6 +207,10 @@ class Replicator
     @config["data_dir"] + format("/%03d/%03d/%03d", sequence / 1000000, (sequence / 1000) % 1000, (sequence % 1000))
   end
 
+  def s3_stem
+    @config["s3_dir"] + format("/%03d/%03d/%03d", sequence / 1000000, (sequence / 1000) % 1000, (sequence % 1000))
+  end
+
   def write_tmp_files!(changesets)
     data_file = data_stem + ".osm.gz"
     tmp_state = @config["state_file"] + ".tmp"
@@ -255,6 +259,14 @@ class Replicator
     # from the dirent and has been updated to account for any allocations.
     fdirsync(File.dirname(data_file))
     fdirsync(File.dirname(@config["state_file"]))
+
+    if @config["s3_dir"]
+      s3_file = s3_stem + ".osm.gz"
+      s3_state_file = s3_stem + ".state.txt"
+
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_file, s3_file)
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_state_file, s3_state_file)
+    end
   end
 
   # saves new state (including the changeset dump xml)
diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb
index d5a8485d7..0a25ad959 100644
--- a/cookbooks/planet/recipes/replication.rb
+++ b/cookbooks/planet/recipes/replication.rb
@@ -208,6 +208,7 @@ systemd_service "replication-changesets" do
   exec_start "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
   sandbox :enable_network => true
   read_write_paths [
+    "/home/planet/.aws",
     "/run/replication",
     "/store/planet/replication/changesets"
   ]
diff --git a/cookbooks/planet/templates/default/changesets.conf.erb b/cookbooks/planet/templates/default/changesets.conf.erb
index 48422054f..752b21566 100644
--- a/cookbooks/planet/templates/default/changesets.conf.erb
+++ b/cookbooks/planet/templates/default/changesets.conf.erb
@@ -1,4 +1,5 @@
 state_file: /store/planet/replication/changesets/state.yaml
 db: host=<%= node[:web][:database_host] %> dbname=openstreetmap user=planetdiff password=<%= @password %>
 data_dir: /store/planet/replication/changesets
+s3_dir: s3://osm-planet-eu-central-1/changesets/replication/minute
 lock_file: /run/replication/changesets.lock
-- 
2.39.5