Add support for using osm2pgsql-replication on render servers

diff --git a/cookbooks/systemd/resources/service.rb b/cookbooks/systemd/resources/service.rb
index b8a58a88f1c9e8d8ff93e91a5621072bf48cff2c..8cd4406cf5377b964860054f917c7b4478926da5 100644 (file)
--- a/cookbooks/systemd/resources/service.rb
+++ b/cookbooks/systemd/resources/service.rb
@@ -71,6 +71,7 @@ property :standard_error, String,
 property :success_exit_status, [Integer, String, Array]
 property :restart, String,
          :is => %w[on-success on-failure on-abnormal on-watchdog on-abort always]
 property :success_exit_status, [Integer, String, Array]
 property :restart, String,
          :is => %w[on-success on-failure on-abnormal on-watchdog on-abort always]

+property :restart_sec, [Integer, String]

 property :protect_proc, String,
          :is => %w[noaccess invisible ptraceable default]
 property :proc_subset, String,
 property :protect_proc, String,
          :is => %w[noaccess invisible ptraceable default]
 property :proc_subset, String,

diff --git a/cookbooks/systemd/templates/default/service.erb b/cookbooks/systemd/templates/default/service.erb
index 01e9e1d4fa89864bf1f6ab3a6655a2dd048f04aa..29bd10e3380a4b3853bda20cacc01339bf222f31 100644 (file)
--- a/cookbooks/systemd/templates/default/service.erb
+++ b/cookbooks/systemd/templates/default/service.erb
@@ -262,6 +262,9 @@ SuccessExitStatus=<%= Array(@success_exit_status).join(" ") %>
 <% if @restart -%>
 Restart=<%= @restart %>
 <% end -%>
 <% if @restart -%>
 Restart=<%= @restart %>
 <% end -%>

+<% if @restart_sec -%>
+RestartSec=<%= @restart_sec %>
+<% end -%>

 <% if @timeout_start_sec -%>
 TimeoutStartSec=<%= @timeout_start_sec %>
 <% end -%>
 <% if @timeout_start_sec -%>
 TimeoutStartSec=<%= @timeout_start_sec %>
 <% end -%>

diff --git a/cookbooks/tile/attributes/default.rb b/cookbooks/tile/attributes/default.rb
index 776e274b08e11c7757833af9a347088a52e72f04..cbdf35bf5c2456e0adcfb5f1dc6111a6d4613912 100644 (file)
--- a/cookbooks/tile/attributes/default.rb
+++ b/cookbooks/tile/attributes/default.rb
@@ -8,6 +8,7 @@ default[:tile][:database][:tag_transform_script] = nil
 
 default[:tile][:mapnik] = "3.1"
 
 
 default[:tile][:mapnik] = "3.1"
 

+default[:tile][:replication][:engine] = "custom"

 default[:tile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute"
 
 default[:tile][:data] = {}
 default[:tile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute"
 
 default[:tile][:data] = {}

diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb
index f693f43825dea12141be80328938a2b66683ea67..bfedc3c170ff4ef2bf239fe6269d46f7e89ea3ed 100644 (file)
--- a/cookbooks/tile/recipes/default.rb
+++ b/cookbooks/tile/recipes/default.rb
@@ -522,77 +522,134 @@ remote_directory "/usr/local/bin" do
   files_mode "755"
 end
 
   files_mode "755"
 end
 

-template "/usr/local/bin/expire-tiles" do
-  source "expire-tiles.erb"
-  owner "root"
-  group "root"
-  mode "755"
-end
-

 directory "/var/lib/replicate" do
   owner "tile"
   group "tile"
   mode "755"
 end
 
 directory "/var/lib/replicate" do
   owner "tile"
   group "tile"
   mode "755"
 end
 

-directory "/var/lib/replicate/expire-queue" do
-  owner "tile"
-  group "_renderd"
-  mode "775"
-end
+if node[:tile][:replication][:engine] == "custom"
+  template "/usr/local/bin/expire-tiles" do
+    source "expire-tiles-custom.erb"
+    owner "root"
+    group "root"
+    mode "755"
+  end

 
 

-template "/usr/local/bin/replicate" do
-  source "replicate.erb"
-  owner "root"
-  group "root"
-  mode "755"
-  variables :postgresql_version => postgresql_version.to_f
-end
+  directory "/var/lib/replicate/expire-queue" do
+    owner "tile"
+    group "_renderd"
+    mode "775"
+  end

 
 

-systemd_service "expire-tiles" do
-  description "Tile dirtying service"
-  type "simple"
-  user "_renderd"
-  exec_start "/usr/local/bin/expire-tiles"
-  nice 10
-  sandbox true
-  read_write_paths tile_directories + [
-    "/store/database/nodes",
-    "/var/lib/replicate/expire-queue",
-    "/var/log/tile"
-  ]
-end
+  template "/usr/local/bin/replicate" do
+    source "replicate.erb"
+    owner "root"
+    group "root"
+    mode "755"
+    variables :postgresql_version => postgresql_version.to_f
+  end

 
 

-systemd_path "expire-tiles" do
-  description "Tile dirtying trigger"
-  directory_not_empty "/var/lib/replicate/expire-queue"
-end
+  systemd_service "expire-tiles" do
+    description "Tile dirtying service"
+    type "simple"
+    user "_renderd"
+    exec_start "/usr/local/bin/expire-tiles"
+    nice 10
+    sandbox true
+    read_write_paths tile_directories + [
+                       "/store/database/nodes",
+                       "/var/lib/replicate/expire-queue",
+                       "/var/log/tile"
+                     ]
+  end

 
 

-service "expire-tiles.path" do
-  action [:enable, :start]
-  subscribes :restart, "systemd_path[expire-tiles]"
-end
+  systemd_path "expire-tiles" do
+    description "Tile dirtying trigger"
+    directory_not_empty "/var/lib/replicate/expire-queue"
+  end

 
 

-systemd_service "replicate" do
-  description "Rendering database replication service"
-  after "postgresql.service"
-  wants "postgresql.service"
-  user "tile"
-  exec_start "/usr/local/bin/replicate"
-  sandbox :enable_network => true
-  restrict_address_families "AF_UNIX"
-  read_write_paths [
-    "/store/database/nodes",
-    "/var/lib/replicate",
-    "/var/log/tile"
+  service "expire-tiles.path" do
+    action [:enable, :start]
+    subscribes :restart, "systemd_path[expire-tiles]"
+  end
+
+  systemd_service "replicate" do
+    description "Rendering database replication service"
+    after "postgresql.service"
+    wants "postgresql.service"
+    user "tile"
+    exec_start "/usr/local/bin/replicate"
+    sandbox :enable_network => true
+    restrict_address_families "AF_UNIX"
+    read_write_paths [
+      "/store/database/nodes",
+      "/var/lib/replicate",
+      "/var/log/tile"
+    ]
+    restart "on-failure"
+  end
+
+  service "replicate" do
+    action [:enable, :start]
+    subscribes :restart, "template[/usr/local/bin/replicate]"
+    subscribes :restart, "systemd_service[replicate]"
+  end
+elsif node[:tile][:replication][:engine] == "osm2pgsql"
+  service "expire-tiles.path" do
+    action [:disable, :stop]
+  end
+
+  systemd_path "expire-tiles" do
+    action :delete
+  end
+
+  service "expire-tiles" do
+    action :stop
+  end
+
+  systemd_service "expire-tiles" do
+    action :delete
+  end
+
+  template "/usr/local/bin/expire-tiles" do
+    source "expire-tiles-osm2pgsql.erb"
+    owner "root"
+    group "root"
+    mode "755"
+  end
+
+  osm2pgsql_arguments = %w[
+    --number-processes=1
+    --log-progress=false
+    --expire-tiles=13-19
+    --expire-output=/var/lib/replicate/dirty-tiles.txt

   ]
   ]

-  restart "on-failure"
-end

 
 

-service "replicate" do
-  action [:enable, :start]
-  subscribes :restart, "template[/usr/local/bin/replicate]"
-  subscribes :restart, "systemd_service[replicate]"
+  osm2pgsql_arguments.append("--multi-geometry") if node[:tile][:database][:multi_geometry]
+  osm2pgsql_arguments.append("--hstore") if node[:tile][:database][:hstore]
+  osm2pgsql_arguments.append("--tag-transform-script=#{node[:tile][:database][:tag_transform_script]}") if node[:tile][:database][:tag_transform_script]
+
+  systemd_service "replicate" do
+    description "Rendering database replication service"
+    after "postgresql.service"
+    wants "postgresql.service"
+    user "tile"
+    exec_start "/bin/osm2pgsql-replication update --database gis --post-processing /usr/local/bin/expire-tiles -- #{osm2pgsql_arguments.join(' ')}"
+    sandbox :enable_network => true
+    restrict_address_families "AF_UNIX"
+    read_write_paths [
+      "/store/database/nodes",
+      "/var/lib/replicate"
+    ]
+    restart "always"
+    restart_sec 30
+  end
+
+  service "replicate" do
+    action [:enable, :start]
+    subscribes :restart, "systemd_service[replicate]"
+  end

 end
 
 template "/usr/local/bin/render-lowzoom" do
 end
 
 template "/usr/local/bin/render-lowzoom" do

diff --git a/cookbooks/tile/templates/default/expire-tiles.erb b/cookbooks/tile/templates/default/expire-tiles-custom.erb
similarity index 100%
rename from cookbooks/tile/templates/default/expire-tiles.erb
rename to cookbooks/tile/templates/default/expire-tiles-custom.erb

diff --git a/cookbooks/tile/templates/default/expire-tiles-osm2pgsql.erb b/cookbooks/tile/templates/default/expire-tiles-osm2pgsql.erb
new file mode 100644 (file)
index 0000000..fd2469a
--- /dev/null
+++ b/cookbooks/tile/templates/default/expire-tiles-osm2pgsql.erb
@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+render_expired \
+  --map=default \
+  --socket=/var/run/renderd/renderd.sock \
+  --tiledir=/srv/tile.openstreetmap.org/tiles \
+  --min-zoom=13 --max-zoom=19 \
+  --touch-from=13 < /var/lib/replicate/dirty-tiles.txt
+
+rm -f /var/lib/replicate/dirty-tiles.txt