Added Luxembourg 2019 lidar hillshade

[chef.git] / cookbooks / tile / templates / default / replicate.erb

diff --git a/cookbooks/tile/templates/default/replicate.erb b/cookbooks/tile/templates/default/replicate.erb
index 5d940375bf36e60d6e96a19c5b5692fc6b3e6a1e..bdd271f61a044a2c8474e3a22f6be99df3f78ff6 100644 (file)
--- a/cookbooks/tile/templates/default/replicate.erb
+++ b/cookbooks/tile/templates/default/replicate.erb
@@ -2,76 +2,111 @@
 
 # DO NOT EDIT - This file is being maintained by Chef
 
 
 # DO NOT EDIT - This file is being maintained by Chef
 

-# Initialize timestamp with day of latest planet dump
-# Setting to midnight ensures we get conistent data after first run
-# osmosis --read-replication-interval-init
+# Before running updates, the replication needs to be set up with the timestamp
+# set to the day of the latest planet dump. Setting to midnight ensures we get
+# conistent data after first run. osmosis --read-replication-interval-init is
+# used to initially create the state file
+
+# Define exit handler
+function onexit {
+    [ -f sequence-prev.txt ] && mv sequence-prev.txt sequence.txt
+}

 
 # Send output to the log
 exec > /var/log/tile/replicate.log 2>&1
 
 # Change to the replication state directory
 cd /var/lib/replicate
 
 # Send output to the log
 exec > /var/log/tile/replicate.log 2>&1
 
 # Change to the replication state directory
 cd /var/lib/replicate

+<% if @postgresql_version >= 12 -%>
+
+# Disable JIT and parallel workers in postgres
+export PGOPTIONS="-c jit=off -c max_parallel_workers_per_gather=0"
+<% end -%>

 
 

-# Read in initial state
-. state.txt
+# Install exit handler
+trap onexit EXIT

 
 # Loop indefinitely
 while true
 do
     # Work out the name of the next file
 
 # Loop indefinitely
 while true
 do
     # Work out the name of the next file

-    file="changes-${sequenceNumber}.osm.gz"
+    file="changes-$(cat sequence.txt).osc.gz"
+
+    # Save sequence file so we can rollback if an error occurs
+    cp sequence.txt sequence-prev.txt

 
     # Fetch the next set of changes
 
     # Fetch the next set of changes

-    osmosis --read-replication-interval --simc --write-xml-change file="${file}" compressionMethod="gzip"
+    pyosmium-get-changes --sequence-file=sequence.txt --outfile=${file}
+
+    # Save exit status
+    status=$?

 
     # Check for errors
 
     # Check for errors

-    if [ $? -eq 0 ]
+    if [ $status -eq 0 ]

     then
         # Enable exit on error
     then
         # Enable exit on error

-       set -e
+        set -e

 
 

-        # Remember the previous sequence number
-       prevSequenceNumber=$sequenceNumber
+        # Log the new data
+        echo "Fetched new data from $(cat sequence-prev.txt) to $(cat sequence.txt) into ${file}"

 
 

-        # Read in new state
-       . state.txt
+        # Apply the changes to the database
+        osm2pgsql --database gis --slim --append --number-processes=1 \
+<% if node[:tile][:database][:node_file] -%>
+                  --flat-nodes=<%= node[:tile][:database][:node_file] %> \
+<% end -%>
+<% if node[:tile][:database][:multi_geometry] -%>
+                  --multi-geometry \
+<% end -%>
+<% if node[:tile][:database][:hstore] -%>
+                  --hstore \
+<% end -%>
+<% if node[:tile][:database][:style_file] -%>
+                  --style=<%= node[:tile][:database][:style_file] %> \
+<% end -%>
+<% if node[:tile][:database][:tag_transform_script] -%>
+                  --tag-transform-script=<%= node[:tile][:database][:tag_transform_script] %> \
+<% end -%>
+                  ${file}
+
+        # No need to rollback now
+        rm sequence-prev.txt

 
 

-        # Did we get any new data?
-       if [ "${sequenceNumber}" == "${prevSequenceNumber}" ]
+        # Get buffer count
+        buffers=$(osmium fileinfo --extended --get=data.buffers.count ${file})
+
+        # If this diff has content mark it as the latest diff
+        if [ $buffers -gt 0 ]

         then
         then

-            # Log the lack of data
-           echo "No new data available. Sleeping..."
-
-           # Remove file, it will just be an empty changeset
-           rm ${file}
-
-            # Sleep for a short while
-           sleep 30
-       else
-            # Log the new data
-           echo "Fetched new data from ${prevSequenceNumber} to ${sequenceNumber} into ${file}"
-
-            # Apply the changes to the database
-<% if node[:tile][:node_file] -%>
-            osm2pgsql --slim --append --flat-nodes=<%= node[:tile][:node_file] %> ${file}
-<% else -%>
-            osm2pgsql --slim --append ${file}
-<% end -%>
+            ln -f ${file} changes-latest.osc.gz
+        fi

 
 

-            # Expire tiles which are touched by the changes
-            sudo -u www-data /usr/local/bin/expire-tiles ${file} > /dev/null 2>&1 &
-       fi
+        # Queue these changes for expiry processing
+        ln ${file} expire-queue/${file}

 
         # Delete old downloads
 
         # Delete old downloads

-       find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \;
+        find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \;

 
         # Disable exit on error
         set +e
 
         # Disable exit on error
         set +e

+    elif [ $status -eq 3 ]
+    then
+        # Log the lack of data
+        echo "No new data available. Sleeping..."
+
+        # Remove file, it will just be an empty changeset
+        rm ${file}
+
+        # Sleep for a short while
+        sleep 30

     else
         # Log our failure to fetch changes
     else
         # Log our failure to fetch changes

-       echo "Failed to fetch changes - waiting a few minutes before retry"
+        echo "Failed to fetch changes - waiting a few minutes before retry"
+
+        # Remove any output that was produced
+        rm -f ${file}

 
         # Wait five minutes and have another go
 
         # Wait five minutes and have another go

-       sleep 300
+        sleep 300

     fi
 done
     fi
 done