X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/af30bf6e1e06bfc20250818d65a8216ca5dce634..9198700edaa62858cbc8f07cc883b2d4283831a6:/cookbooks/tile/templates/default/replicate.erb?ds=sidebyside diff --git a/cookbooks/tile/templates/default/replicate.erb b/cookbooks/tile/templates/default/replicate.erb index e1ecab695..eedf4e576 100644 --- a/cookbooks/tile/templates/default/replicate.erb +++ b/cookbooks/tile/templates/default/replicate.erb @@ -2,13 +2,14 @@ # DO NOT EDIT - This file is being maintained by Chef -# Initialize timestamp with day of latest planet dump -# Setting to midnight ensures we get conistent data after first run -# osmosis --read-replication-interval-init +# Before running updates, the replication needs to be set up with the timestamp +# set to the day of the latest planet dump. Setting to midnight ensures we get +# conistent data after first run. osmosis --read-replication-interval-init is +# used to initially create the state file # Define exit handler function onexit { - [ -f state-prev.txt ] && mv state-prev.txt state.txt + [ -f sequence-prev.txt ] && mv sequence-prev.txt sequence.txt } # Send output to the log @@ -20,75 +21,87 @@ cd /var/lib/replicate # Install exit handler trap onexit EXIT -# Read in initial state -. state.txt - # Loop indefinitely while true do # Work out the name of the next file - file="changes-${sequenceNumber}.osm.gz" + file="changes-$(cat sequence.txt).osc.gz" - # Save state file so we can rollback if an error occurs - cp state.txt state-prev.txt + # Save sequence file so we can rollback if an error occurs + cp sequence.txt sequence-prev.txt # Fetch the next set of changes - osmosis --read-replication-interval --simc --write-xml-change file="${file}" compressionMethod="gzip" + pyosmium-get-changes --server=<%= node[:tile][:replication][:url] %> --sequence-file=sequence.txt --outfile=${file} + + # Save exit status + status=$? # Check for errors - if [ $? -eq 0 ] + if [ $status -eq 0 ] then # Enable exit on error - set -e + set -e - # Remember the previous sequence number - prevSequenceNumber=$sequenceNumber + # Log the new data + echo "Fetched new data from $(cat sequence-prev.txt) to $(cat sequence.txt) into ${file}" - # Read in new state - . state.txt - - # Did we get any new data? - if [ "${sequenceNumber}" == "${prevSequenceNumber}" ] - then - # Log the lack of data - echo "No new data available. Sleeping..." - - # Remove file, it will just be an empty changeset - rm ${file} - - # No need to rollback now - rm state-prev.txt - - # Sleep for a short while - sleep 30 - else - # Log the new data - echo "Fetched new data from ${prevSequenceNumber} to ${sequenceNumber} into ${file}" - - # Apply the changes to the database -<% if node[:tile][:node_file] -%> - osm2pgsql --slim --append --flat-nodes=<%= node[:tile][:node_file] %> ${file} -<% else -%> - osm2pgsql --slim --append ${file} + # Apply the changes to the database + osm2pgsql --database gis --slim --append --number-processes=1 \ +<% if node[:tile][:database][:node_file] -%> + --flat-nodes=<%= node[:tile][:database][:node_file] %> \ +<% end -%> +<% if node[:tile][:database][:multi_geometry] -%> + --multi-geometry \ +<% end -%> +<% if node[:tile][:database][:hstore] -%> + --hstore \ +<% end -%> +<% if node[:tile][:database][:style_file] -%> + --style=<%= node[:tile][:database][:style_file] %> \ +<% end -%> +<% if node[:tile][:database][:tag_transform_script] -%> + --tag-transform-script=<%= node[:tile][:database][:tag_transform_script] %> \ <% end -%> + ${file} - # No need to rollback now - rm state-prev.txt + # No need to rollback now + rm sequence-prev.txt - # Expire tiles which are touched by the changes - sudo -u www-data /usr/local/bin/expire-tiles ${file} > /dev/null 2>&1 & - fi + # Get buffer count + buffers=$(osmium fileinfo --extended --get=data.buffers.count ${file}) + + # If this diff has content mark it as the latest diff + if [ $buffers -gt 0 ] + then + ln -f ${file} changes-latest.osc.gz + fi + + # Queue these changes for expiry processing + ln ${file} expire-queue/${file} # Delete old downloads - find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \; + find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \; # Disable exit on error set +e + elif [ $status -eq 3 ] + then + # Log the lack of data + echo "No new data available. Sleeping..." + + # Remove file, it will just be an empty changeset + rm ${file} + + # Sleep for a short while + sleep 30 else # Log our failure to fetch changes - echo "Failed to fetch changes - waiting a few minutes before retry" + echo "Failed to fetch changes - waiting a few minutes before retry" + + # Remove any output that was produced + rm -f ${file} # Wait five minutes and have another go - sleep 300 + sleep 300 fi done