Copy changeset replication diffs to S3

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/files/default/replication-bin/replicate-changesets
index 8731c0d207a5e4f759c7e36de9821fec4ad859e4..87334f21a06b51f840b01e708a6a03c4fce48a71 100755 (executable)
--- a/cookbooks/planet/files/default/replication-bin/replicate-changesets
+++ b/cookbooks/planet/files/default/replication-bin/replicate-changesets
@@ -207,6 +207,10 @@ class Replicator
     @config["data_dir"] + format("/%03d/%03d/%03d", sequence / 1000000, (sequence / 1000) % 1000, (sequence % 1000))
   end
 
+  def s3_stem
+    @config["s3_dir"] + format("/%03d/%03d/%03d", sequence / 1000000, (sequence / 1000) % 1000, (sequence % 1000))
+  end
+
   def write_tmp_files!(changesets)
     data_file = data_stem + ".osm.gz"
     tmp_state = @config["state_file"] + ".tmp"
@@ -255,6 +259,14 @@ class Replicator
     # from the dirent and has been updated to account for any allocations.
     fdirsync(File.dirname(data_file))
     fdirsync(File.dirname(@config["state_file"]))
+
+    if @config["s3_dir"]
+      s3_file = s3_stem + ".osm.gz"
+      s3_state_file = s3_stem + ".state.txt"
+
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_file, s3_file)
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_state_file, s3_state_file)
+    end
   end
 
   # saves new state (including the changeset dump xml)

diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb
index d5a8485d7efade32f704036b92df4c2b72146278..0a25ad959bc44cc9756f5c0fbfd477543b8eed3c 100644 (file)
--- a/cookbooks/planet/recipes/replication.rb
+++ b/cookbooks/planet/recipes/replication.rb
@@ -208,6 +208,7 @@ systemd_service "replication-changesets" do
   exec_start "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
   sandbox :enable_network => true
   read_write_paths [
+    "/home/planet/.aws",
     "/run/replication",
     "/store/planet/replication/changesets"
   ]

diff --git a/cookbooks/planet/templates/default/changesets.conf.erb b/cookbooks/planet/templates/default/changesets.conf.erb
index 48422054f4459fc8f72895d973a07659437c0f5e..752b2156635ef7591a3aa9551561b0f328e646b7 100644 (file)
--- a/cookbooks/planet/templates/default/changesets.conf.erb
+++ b/cookbooks/planet/templates/default/changesets.conf.erb
@@ -1,4 +1,5 @@
 state_file: /store/planet/replication/changesets/state.yaml
 db: host=<%= node[:web][:database_host] %> dbname=openstreetmap user=planetdiff password=<%= @password %>
 data_dir: /store/planet/replication/changesets
+s3_dir: s3://osm-planet-eu-central-1/changesets/replication/minute
 lock_file: /run/replication/changesets.lock