Merge remote-tracking branch 'github/pull/678'

[chef.git] / cookbooks / tile / templates / default / replicate.erb

diff --git a/cookbooks/tile/templates/default/replicate.erb b/cookbooks/tile/templates/default/replicate.erb
index bdd271f61a044a2c8474e3a22f6be99df3f78ff6..8d8e287925dac83e13106fd4080bf96ccdd62477 100644 (file)
--- a/cookbooks/tile/templates/default/replicate.erb
+++ b/cookbooks/tile/templates/default/replicate.erb
@@ -1,112 +1,27 @@
-#!/bin/bash
+#!/bin/sh

 
 # DO NOT EDIT - This file is being maintained by Chef
 
 
 # DO NOT EDIT - This file is being maintained by Chef
 

-# Before running updates, the replication needs to be set up with the timestamp
-# set to the day of the latest planet dump. Setting to midnight ensures we get
-# conistent data after first run. osmosis --read-replication-interval-init is
-# used to initially create the state file
-
-# Define exit handler
-function onexit {
-    [ -f sequence-prev.txt ] && mv sequence-prev.txt sequence.txt
-}
-
-# Send output to the log
-exec > /var/log/tile/replicate.log 2>&1
-
-# Change to the replication state directory
-cd /var/lib/replicate
-<% if @postgresql_version >= 12 -%>
-
-# Disable JIT and parallel workers in postgres
-export PGOPTIONS="-c jit=off -c max_parallel_workers_per_gather=0"
-<% end -%>
-
-# Install exit handler
-trap onexit EXIT
-
-# Loop indefinitely
-while true
+while [ 1 = 1 ]

 do
 do

-    # Work out the name of the next file
-    file="changes-$(cat sequence.txt).osc.gz"
-
-    # Save sequence file so we can rollback if an error occurs
-    cp sequence.txt sequence-prev.txt
-
-    # Fetch the next set of changes
-    pyosmium-get-changes --sequence-file=sequence.txt --outfile=${file}
-
-    # Save exit status
-    status=$?
-
-    # Check for errors
-    if [ $status -eq 0 ]
-    then
-        # Enable exit on error
-        set -e
-
-        # Log the new data
-        echo "Fetched new data from $(cat sequence-prev.txt) to $(cat sequence.txt) into ${file}"
-
-        # Apply the changes to the database
-        osm2pgsql --database gis --slim --append --number-processes=1 \
-<% if node[:tile][:database][:node_file] -%>
-                  --flat-nodes=<%= node[:tile][:database][:node_file] %> \
-<% end -%>
+  /bin/osm2pgsql-replication update \
+    --database gis \
+    --post-processing /usr/local/bin/replicate-post \
+    -- \
+    --number-processes=1 \
+    --log-progress=false \
+    --expire-tiles=10-16 \
+    --expire-segment-length=90000 \
+    --expire-output=/var/lib/replicate/dirty-tiles.txt \

 <% if node[:tile][:database][:multi_geometry] -%>
 <% if node[:tile][:database][:multi_geometry] -%>

-                  --multi-geometry \
+    --multi-geometry \

 <% end -%>
 <% if node[:tile][:database][:hstore] -%>
 <% end -%>
 <% if node[:tile][:database][:hstore] -%>

-                  --hstore \
-<% end -%>
-<% if node[:tile][:database][:style_file] -%>
-                  --style=<%= node[:tile][:database][:style_file] %> \
+    --hstore \

 <% end -%>
 <% if node[:tile][:database][:tag_transform_script] -%>
 <% end -%>
 <% if node[:tile][:database][:tag_transform_script] -%>

-                  --tag-transform-script=<%= node[:tile][:database][:tag_transform_script] %> \
+    --tag-transform-script=<%= node[:tile][:database][:tag_transform_script] %>

 <% end -%>
 <% end -%>

-                  ${file}
-
-        # No need to rollback now
-        rm sequence-prev.txt
-
-        # Get buffer count
-        buffers=$(osmium fileinfo --extended --get=data.buffers.count ${file})
-
-        # If this diff has content mark it as the latest diff
-        if [ $buffers -gt 0 ]
-        then
-            ln -f ${file} changes-latest.osc.gz
-        fi
-
-        # Queue these changes for expiry processing
-        ln ${file} expire-queue/${file}
-
-        # Delete old downloads
-        find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \;
-
-        # Disable exit on error
-        set +e
-    elif [ $status -eq 3 ]
-    then
-        # Log the lack of data
-        echo "No new data available. Sleeping..."
-
-        # Remove file, it will just be an empty changeset
-        rm ${file}
-
-        # Sleep for a short while
-        sleep 30
-    else
-        # Log our failure to fetch changes
-        echo "Failed to fetch changes - waiting a few minutes before retry"
-
-        # Remove any output that was produced
-        rm -f ${file}

 
 

-        # Wait five minutes and have another go
-        sleep 300
-    fi
+    sleep 30

 done
 done