Re-render vector tiles by layer

This should reduce the amount of work that is caused by updates.

diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb
index feda712a0ba3f0599b35b5ca4adad035eab64463..0f4041565e90f1a921e6807362e590a0240cf411 100644 (file)
--- a/cookbooks/vectortile/recipes/default.rb
+++ b/cookbooks/vectortile/recipes/default.rb
@@ -253,12 +253,14 @@ template "/usr/local/bin/vector-update" do
   variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender"
 end
 
   variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender"
 end
 

+rerender_layers = %w[addresses boundaries bridges buildings land pois public_transport sites street_polygons streets water_lines_labels water_lines water_polygons].join(" ")
+

 template "/usr/local/bin/tiles-rerender" do
   source "tiles-rerender.erb"
   owner "root"
   group "root"
   mode "755"
 template "/usr/local/bin/tiles-rerender" do
   source "tiles-rerender.erb"
   owner "root"
   group "root"
   mode "755"

-  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4
+  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4, :layers => "#{rerender_layers}"

 end
 
 systemd_service "replicate" do
 end
 
 systemd_service "replicate" do

diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb
index 6c01b27f6f4a7fec493fd2704e09ffc2b5bb3456..d1ca4bdab250841ec1f6d232ba3f0969abf68afe 100644 (file)
--- a/cookbooks/vectortile/templates/default/tiles-rerender.erb
+++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb
@@ -2,9 +2,12 @@
 set -eu
 
 cd "<%= @expiry_dir %>"
 set -eu
 
 cd "<%= @expiry_dir %>"

-
+# Only expire z10+ tiles
+rm -f z?-*.txt

 wc -l z*.txt
 wc -l z*.txt

-cat z*.txt | <%= @tilekiln_bin %> generate tiles \
+(for layer in <%= @layers %>; do
+  find . -type f -name "z*-${layer}.txt" -execdir sed -e "s/\$/,${layer}/" {} \;
+done) | <%= @tilekiln_bin %> generate layers \

 --source-dbname "<%= @source_database %>" \
 --storage-dbname "<%= @storage_database %>" \
 --num-threads "<%= node[:vectortile][:replication][:threads] %>" \
 --source-dbname "<%= @source_database %>" \
 --storage-dbname "<%= @storage_database %>" \
 --num-threads "<%= node[:vectortile][:replication][:threads] %>" \

diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb
index ba708c9b836690dbdcdd74ff551742c4dff2715d..a356cbb10e342706fff5093c9ec60b99dca47ed2 100644 (file)
--- a/cookbooks/vectortile/templates/default/vector-update-tile.erb
+++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb
@@ -8,6 +8,8 @@ set -eu
 export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;'
 
 cd "<%= @expiry_dir %>"
 export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;'
 
 cd "<%= @expiry_dir %>"

+rm -f z*.txt
+

 osm2pgsql-replication update \
   -d "<%= @source_database %>" \
   --max-diff-size "<%= @diff_size %>" \
 osm2pgsql-replication update \
   -d "<%= @source_database %>" \
   --max-diff-size "<%= @diff_size %>" \