nominatim: clamp down on rate limits

[chef.git] / cookbooks / nominatim / templates / default / clean-db-nominatim.erb

diff --git a/cookbooks/nominatim/templates/default/clean-db-nominatim.erb b/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
index 55c2314be9d48ab3ffd492562b21969ff57def58..921a60d096fe7180717fcf2f7943d3a75ea62472 100644 (file)
--- a/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
+++ b/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
@@ -3,18 +3,40 @@
 # DO NOT EDIT - This file is being maintained by Chef
 
 
-# Vaccum all tables with indices on integer arrays.
-# Agressive vacuuming seems to help against index bloat.
-psql -q -d nominatim -c 'VACUUM ANALYSE search_name'
-psql -q -d nominatim -c 'VACUUM ANALYSE search_name_country'
+# Cleanup archive directory
+oldest=`date +%s`
 
-for i in `seq 0 246`; do
-  psql -q -d nominatim -c "VACUUM ANALYSE search_name_${i}"
+for fl in <%= @streaming_clients %>; do
+  lastupdate=`wget -O - -q https://${fl}.openstreetmap.org/last_update.php`
+  if [[ "$?" != "0" ]]; then
+    oldest=0
+    break
+  else
+    epoch=`date +%s -d "$lastupdate"`
+    if [[ "$epoch" -lt "$oldest" ]]; then
+        oldest=$epoch
+    fi
+  fi
 done
 
-<% if node[:postgresql][:settings][:defaults][:archive_mode] == "on" -%>
 
-# Cleanup archive directory
-find -L /data/postgresql-archive -mtime +1 -print0 | xargs -0r rm
+if [[ "$oldest" != "0" ]]; then
+    now=`date +%s`
+    timeago=$(( ( $now - $oldest ) / 60 + 180 ))
+    if [[ "$timeago" -lt "180" ]]; then
+        echo "Oldest is in the future, something is wrong."
+    else
+      if [[ "$timeago" -le "720" ]]; then
+          timeago=720
+      fi
+
+      find -L <%= @archive_dir %> -mmin +$timeago -print0 | xargs -0r rm
+    fi
+fi
+
+diskfree=`df --output=pcent <%= @archive_dir %> | tail -n 1 | sed s:[^0-9.]::g`
 
-<% end -%>
+if [[ "$diskfree" -lt 10 ]]; then
+    echo "WAL archives running out of space. Stopping updates."
+    touch <%= @update_stop_file %>
+fi