Create a new vectortile role

This role is for tilekiln based vector tile serving

diff --git a/.kitchen.yml b/.kitchen.yml
index 0af8c80de62de90e1df5ccd2585191c9e946e4db..f7cfa5d3838691d95978edc65b8a611749425e28 100644 (file)
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -416,6 +416,9 @@ suites:
   - name: trac
     run_list:
       - recipe[trac::default]
   - name: trac
     run_list:
       - recipe[trac::default]

+  - name: vectortile
+    run_list:
+      - recipe[vectortile::default]

   - name: web-cgimap
     run_list:
       - recipe[web::cgimap]
   - name: web-cgimap
     run_list:
       - recipe[web::cgimap]

diff --git a/cookbooks/postgresql/libraries/postgresql.rb b/cookbooks/postgresql/libraries/postgresql.rb
index 1a39da25a146bc6426ee9d694e1cc0f714da989c..b2df4aed68ef5e3c8998524b2e844efc1e356553 100644 (file)
--- a/cookbooks/postgresql/libraries/postgresql.rb
+++ b/cookbooks/postgresql/libraries/postgresql.rb
@@ -4,6 +4,10 @@ module OpenStreetMap
   class PostgreSQL
     include Chef::Mixin::ShellOut
 
   class PostgreSQL
     include Chef::Mixin::ShellOut
 

+    SCHEMA_PRIVILEGES = [
+      :create, :usage
+    ].freeze
+

     TABLE_PRIVILEGES = [
       :select, :insert, :update, :delete, :truncate, :references, :trigger
     ].freeze
     TABLE_PRIVILEGES = [
       :select, :insert, :update, :delete, :truncate, :references, :trigger
     ].freeze


@@ -115,9 +119,21 @@ module OpenStreetMap
       end
     end
 
       end
     end
 

+    def schemas(database)
+      @schemas ||= {}
+      @schemas[database] ||= query("SELECT n.nspname, pg_catalog.pg_get_userbyid(n.nspowner) AS usename, n.nspacl FROM pg_namespace AS n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema'", :database => database).each_with_object({}) do |schema, schemas|
+        name = "#{schema[:nspname]}"
+
+        schemas[name] = {
+          :owner => schema[:usename],
+          :permissions => parse_acl(schema[:nspacl] || "{}")
+        }
+      end
+    end
+

     def tables(database)
       @tables ||= {}
     def tables(database)
       @tables ||= {}

-      @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind = 'r'", :database => database).each_with_object({}) do |table, tables|
+      @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind IN ('r', 'p')", :database => database).each_with_object({}) do |table, tables|

         name = "#{table[:nspname]}.#{table[:relname]}"
 
         tables[name] = {
         name = "#{table[:nspname]}.#{table[:relname]}"
 
         tables[name] = {

diff --git a/cookbooks/postgresql/resources/schema.rb b/cookbooks/postgresql/resources/schema.rb
new file mode 100644 (file)
index 0000000..a7bf0eb
--- /dev/null
+++ b/cookbooks/postgresql/resources/schema.rb
@@ -0,0 +1,114 @@
+#
+# Cookbook:: postgresql
+# Resource:: postgresql_schema
+#
+# Copyright:: 2024, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+unified_mode true
+
+default_action :create
+
+property :schema, :kind_of => String, :name_property => true
+property :cluster, :kind_of => String, :required => true
+property :database, :kind_of => String, :required => true
+property :owner, :kind_of => String, :required => [:create]
+property :permissions, :kind_of => Hash, :default => {}
+
+action :create do
+  if schemas.include?(qualified_name)
+    # Handle the case of an existing schema
+    if new_resource.owner != schemas[qualified_name][:owner]
+      converge_by("set owner for #{new_resource} to #{new_resource.owner}") do
+        Chef::Log.info("Setting owner for #{new_resource} to #{new_resource.owner}")
+        cluster.execute(:command => "ALTER SCHEMA #{qualified_name} OWNER TO \"#{new_resource.owner}\"", :database => new_resource.database)
+      end
+    end
+
+    schemas[qualified_name][:permissions].each_key do |user|
+      next if new_resource.permissions[user]
+      # Remove permissions from users who no longer have them
+      converge_by("revoke all for #{user} on #{new_resource}") do
+        Chef::Log.info("Revoking all for #{user} on #{new_resource}")
+        cluster.execute(:command => "REVOKE ALL ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database)
+      end
+    end
+    new_resource.permissions.each do |user, new_privileges|
+      current_privileges = schemas[qualified_name][:permissions][user] || {}
+      new_privileges = Array(new_privileges)
+
+      if new_privileges.include?(:all)
+        new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES
+      end
+
+      OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege|
+        if new_privileges.include?(privilege)
+          unless current_privileges.include?(privilege)
+            converge_by("grant #{privilege} for #{user} on #{new_resource}") do
+              Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}")
+              cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database)
+            end
+          end
+        elsif current_privileges.include?(privilege)
+          converge_by("revoke #{privilege} for #{user} on #{new_resource}") do
+            Chef::Log.info("Revoking #{privilege} for #{user} on #{new_resource}")
+            cluster.execute(:command => "REVOKE #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database)
+          end
+        end
+      end
+    end
+  else
+    converge_by "create schema #{new_resource.schema}" do
+      cluster.execute(:command => "CREATE SCHEMA #{new_resource.schema} AUTHORIZATION #{new_resource.owner}", :database => new_resource.database)
+      # Because the schema is new, we don't have to worry about revoking or checking current permissions
+      new_resource.permissions.each do |user, new_privileges|
+        new_privileges = Array(new_privileges)
+        if new_privileges.include?(:all)
+          new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES
+        end
+        OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege|
+          next unless new_privileges.include?(privilege)
+          converge_by("grant #{privilege} for #{user} on #{new_resource}") do
+            Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}")
+            cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database)
+          end
+        end
+      end
+    end
+  end
+end
+
+action :drop do
+  if schemas.include?(qualified_name)
+    converge_by("drop #{new_resource}") do
+      Chef::Log.info("Dropping #{new_resource}")
+      cluster.execute(:command => "DROP SCHEMA #{qualified_name}", :database => new_resource.database)
+    end
+  end
+end
+
+action_class do
+  def cluster
+    @cluster ||= OpenStreetMap::PostgreSQL.new(new_resource.cluster)
+  end
+
+  def schemas
+    @schemas ||= cluster.schemas(new_resource.database)
+  end
+
+  def qualified_name
+    "#{new_resource.name}"
+  end
+end

diff --git a/cookbooks/vectortile/README.md b/cookbooks/vectortile/README.md
new file mode 100644 (file)
index 0000000..659ee7a
--- /dev/null
+++ b/cookbooks/vectortile/README.md
@@ -0,0 +1,9 @@
+# vectortile cookbook
+
+This cookbook installs and configures the tilekiln based tileservers
+
+## Accounts
+The following accounts are used
+- `www-data` for nginx serving static files and proxying
+- `tilekiln` for the process serving tiles
+- `tileupdate` for the process running osm2pgsql and tilekiln on updates

diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb
new file mode 100644 (file)
index 0000000..0df4249
--- /dev/null
+++ b/cookbooks/vectortile/attributes/default.rb
@@ -0,0 +1,15 @@
+default[:vectortile][:database][:cluster] = "16/main"
+default[:vectortile][:database][:postgis] = "3"
+default[:vectortile][:database][:nodes_store] = :flat
+default[:vectortile][:serve][:threads] = node.cpu_cores
+default[:vectortile][:serve][:mode] = :live
+default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute"
+default[:vectortile][:replication][:status] = :enabled
+default[:vectortile][:replication][:tileupdate] = :enabled
+default[:vectortile][:replication][:threads] = node.cpu_cores
+
+default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first]
+default[:postgresql][:monitor_database] = "tiles"
+default[:postgresql][:settings][:defaults][:max_connections] = (node.cpu_cores * 4 + 20).to_s
+default[:accounts][:users][:tileupdate][:status] = :role
+default[:accounts][:users][:tilekiln][:status] = :role

diff --git a/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml b/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml
new file mode 100644 (file)
index 0000000..d60ec1d
--- /dev/null
+++ b/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<access-policy>
+  <cross-domain-access>
+    <policy>
+      <allow-from http-request-headers="*">
+        <domain uri="*"/>
+      </allow-from>
+      <grant-to>
+        <resource path="/" include-subpaths="true"/>
+      </grant-to>
+    </policy>
+  </cross-domain-access>
+</access-policy>

diff --git a/cookbooks/vectortile/files/default/html/crossdomain.xml b/cookbooks/vectortile/files/default/html/crossdomain.xml
new file mode 100644 (file)
index 0000000..35fc434
--- /dev/null
+++ b/cookbooks/vectortile/files/default/html/crossdomain.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0"?>
+<!DOCTYPE cross-domain-policy SYSTEM "http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd">
+<cross-domain-policy>
+    <allow-access-from domain="*" secure="false" />
+</cross-domain-policy>

diff --git a/cookbooks/vectortile/files/default/html/favicon.ico b/cookbooks/vectortile/files/default/html/favicon.ico
new file mode 100644 (file)
index 0000000..975e1cb
Binary files /dev/null and b/cookbooks/vectortile/files/default/html/favicon.ico differ

diff --git a/cookbooks/vectortile/files/default/html/robots.txt b/cookbooks/vectortile/files/default/html/robots.txt
new file mode 100644 (file)
index 0000000..bfb50b0
--- /dev/null
+++ b/cookbooks/vectortile/files/default/html/robots.txt
@@ -0,0 +1,13 @@
+User-agent: *
+Disallow: /12/
+Disallow: /13/
+Disallow: /14/
+Disallow: /15/
+Disallow: /16/
+Disallow: /17/
+Disallow: /18/
+Disallow: /19/
+Disallow: /20/
+Disallow: /21/
+Disallow: /22/
+Disallow: /23/

diff --git a/cookbooks/vectortile/metadata.rb b/cookbooks/vectortile/metadata.rb
new file mode 100644 (file)
index 0000000..d3b44e3
--- /dev/null
+++ b/cookbooks/vectortile/metadata.rb
@@ -0,0 +1,16 @@
+name              "vectortile"
+maintainer        "OpenStreetMap Administrators"
+maintainer_email  "admins@openstreetmap.org"
+license           "Apache-2.0"
+description       "Installs and configures vector tile servers"
+
+version           "1.0.0"
+supports          "ubuntu"
+depends           "accounts"
+depends           "git"
+depends           "nginx"
+depends           "postgresql"
+depends           "prometheus"
+depends           "python"
+depends           "systemd"
+depends           "tools"

diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb
new file mode 100644 (file)
index 0000000..37fe86b
--- /dev/null
+++ b/cookbooks/vectortile/recipes/default.rb
@@ -0,0 +1,282 @@
+#
+# Cookbook:: vectortile
+# Recipe:: default
+#
+# Copyright:: 2024, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "accounts"
+include_recipe "git"
+include_recipe "nginx"
+include_recipe "postgresql"
+include_recipe "prometheus"
+include_recipe "python"
+include_recipe "tools"
+
+directory "/srv/vector.openstreetmap.org" do
+  user "tileupdate"
+  group "tileupdate"
+  mode "755"
+end
+
+nginx_site "default" do
+  action [:delete]
+end
+
+nginx_site "vector.openstreetmap.org" do
+  template "nginx.erb"
+end
+
+ssl_certificate node[:fqdn] do
+  domains [node[:fqdn], "vector.openstreetmap.org"]
+  notifies :reload, "service[nginx]"
+end
+
+remote_directory "/srv/vector.openstreetmap.org/html" do
+  source "html"
+  owner "www-data"
+  group "www-data"
+  mode "755"
+  files_owner "www-data"
+  files_group "www-data"
+  files_mode "644"
+end
+
+template "/srv/vector.openstreetmap.org/html/index.html" do
+  source "index.html.erb"
+  owner "www-data"
+  group "www-data"
+  mode "644"
+end
+
+postgresql_version = node[:vectortile][:database][:cluster].split("/").first
+postgis_version = node[:vectortile][:database][:postgis]
+
+package "postgresql-#{postgresql_version}-postgis-#{postgis_version}"
+
+# Spirit dependencies
+package %w[
+  osm2pgsql
+  gdal-bin
+  python3-yaml
+  python3-psycopg2
+]
+
+style_directory = "/srv/vector.openstreetmap.org/spirit"
+git style_directory do
+  repository "https://github.com/pnorman/spirit.git"
+  user "tileupdate"
+  group "tileupdate"
+end
+
+shortbread_config = "#{style_directory}/shortbread.yaml"
+
+themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark"
+git themepark_directory do
+  repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git"
+  user "tileupdate"
+  group "tileupdate"
+end
+
+tilekiln_directory = "/opt/tilekiln"
+
+python_virtualenv tilekiln_directory do
+  interpreter "/usr/bin/python3"
+end
+
+python_package "tilekiln" do
+  python_virtualenv tilekiln_directory
+  python_version "3"
+  version "0.5.1"
+end
+
+template "/srv/vector.openstreetmap.org/html/index.html" do
+  source "index.html.erb"
+  owner "www-data"
+  group "www-data"
+  mode "644"
+end
+
+directory "/srv/vector.openstreetmap.org/data" do
+  user "tileupdate"
+  group "tileupdate"
+  mode "755"
+end
+
+node_store_options = node[:vectortile][:database][:nodes_store] == :flat ? "--flat-nodes '/srv/vector.openstreetmap.org/data/nodes.bin'" : ""
+template "/usr/local/bin/import-planet" do
+  source "import-planet.erb"
+  owner "root"
+  group "root"
+  mode "755"
+  variables :node_store_options => "#{node_store_options}"
+end
+
+template "/usr/local/bin/tilekiln-storage-init" do
+  source "tilekiln-storage-init.erb"
+  owner "root"
+  group "root"
+  mode "755"
+  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{shortbread_config}"
+end
+
+postgresql_user "tomh" do
+  cluster node[:vectortile][:database][:cluster]
+  superuser true
+end
+
+postgresql_user "pnorman" do
+  cluster node[:vectortile][:database][:cluster]
+  superuser true
+end
+
+postgresql_user "tilekiln" do
+  cluster node[:vectortile][:database][:cluster]
+end
+
+postgresql_user "tileupdate" do
+  cluster node[:vectortile][:database][:cluster]
+end
+
+postgresql_database "tiles" do
+  cluster node[:vectortile][:database][:cluster]
+  owner "tileupdate"
+end
+
+postgresql_database "spirit" do
+  cluster node[:vectortile][:database][:cluster]
+  owner "tileupdate"
+end
+
+postgresql_extension "postgis" do
+  cluster node[:vectortile][:database][:cluster]
+  database "spirit"
+end
+
+postgresql_schema "tilekiln" do
+  cluster node[:vectortile][:database][:cluster]
+  database "tiles"
+  owner "tileupdate"
+  permissions "tileupdate" => :all, "tilekiln" => :usage
+  notifies :run, "execute[tilekiln-storage-init]", :immediately
+end
+
+execute "tilekiln-storage-init" do
+  action :nothing
+  command "/usr/local/bin/tilekiln-storage-init"
+  user "tileupdate"
+end
+
+%w[metadata shortbread_v1].each do |table|
+  postgresql_table table do
+    cluster node[:vectortile][:database][:cluster]
+    database "tiles"
+    schema "tilekiln"
+    owner "tileupdate"
+    permissions "tileupdate" => :all, "tilekiln" => :select
+  end
+end
+
+(0..14).each do |zoom|
+  postgresql_table "shortbread_v1_z#{zoom}" do
+    cluster node[:vectortile][:database][:cluster]
+    database "tiles"
+    schema "tilekiln"
+    owner "tileupdate"
+    permissions "tileupdate" => :all, "tilekiln" => node[:vectortile][:serve][:mode] == :live ? [:select, :insert, :update] : :select
+  end
+end
+
+%w[addresses aerialways aeroways boundaries boundary_labels bridges buildings
+dam_lines dam_polygons ferries land pier_lines pier_polygons place_labels
+planet_osm_nodes planet_osm_rels planet_osm_ways pois public_transport railways
+road_routes roads sites street_polygons streets_labels_points
+streets_polygons_labels water_area_labels water_areas water_lines water_lines_labels].each do |table|
+  postgresql_table table do
+    cluster node[:vectortile][:database][:cluster]
+    database "spirit"
+    schema "public"
+    owner "tileupdate"
+    permissions "tileupdate" => :all, "tilekiln" => :select
+  end
+end
+
+tilekiln_mode = node[:vectortile][:serve][:mode] == :live ? "live --config #{shortbread_config} --source-dbname spirit" : "static"
+
+systemd_service "tilekiln" do
+  description "Tilekiln vector tile server"
+  user "tilekiln"
+  after "postgresql.service"
+  wants "postgresql.service"
+  sandbox :enable_network => true
+  restrict_address_families "AF_UNIX"
+  exec_start "#{tilekiln_directory}/bin/tilekiln serve #{tilekiln_mode} --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]}"
+end
+
+service "tilekiln" do
+  action [:enable, :start]
+end
+
+execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do
+  command "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py -R tilekiln"
+  cwd "/srv/vector.openstreetmap.org/spirit"
+  user "tileupdate"
+  group "tileupdate"
+  ignore_failure true
+end
+
+template "/usr/local/bin/vector-update" do
+  source node[:vectortile][:replication][:tileupdate] == :enabled ? "vector-update-tile.erb" : "vector-update-notile.erb"
+  owner "root"
+  group "root"
+  mode "755"
+  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :post_processing => "/usr/local/bin/tiles-rerender"
+end
+
+template "/usr/local/bin/tiles-rerender" do
+  source "tiles-rerender.erb"
+  owner "root"
+  group "root"
+  mode "755"
+  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :update_threads => 4
+end
+
+systemd_service "replicate" do
+  description "Get replication updates"
+  user "tileupdate"
+  after "postgresql.service"
+  wants "postgresql.service"
+  sandbox :enable_network => true
+  restrict_address_families "AF_UNIX"
+  read_write_paths ["/srv/vector.openstreetmap.org/data/"]
+  exec_start "/usr/local/bin/vector-update"
+end
+
+systemd_timer "replicate" do
+  description "Get replication updates"
+  on_boot_sec 60
+  on_unit_active_sec 30
+  accuracy_sec 5
+end
+
+if node[:vectortile][:replication][:status] == :enabled
+  service "replicate.timer" do
+    action [:enable, :start]
+  end
+else
+  service "replicate.timer" do
+    action [:stop, :disable]
+  end
+end

diff --git a/cookbooks/vectortile/templates/default/import-planet.erb b/cookbooks/vectortile/templates/default/import-planet.erb
new file mode 100644 (file)
index 0000000..bffa34a
--- /dev/null
+++ b/cookbooks/vectortile/templates/default/import-planet.erb
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Usage
+# sudo -u tileupdate import-planet osmfile.osm.pbf
+# sudo -u tileupdate import-planet --cache 50000 osmfile.osm.pbf
+# This script sets the appropriate Lua and style paths for the osm2pgsql import
+
+set -e
+
+export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;'
+
+# Import the osm2pgsql file specified as an argument, using the locations for spirit
+osm2pgsql \
+  --output flex \
+  --style '/srv/vector.openstreetmap.org/spirit/shortbread.lua' \
+  --slim \
+  <%= @node_store_options %> \
+  -d spirit \
+  $@
+
+# Set up replication. This doesn't specify the replication server, so it will use planet.osm.org on extracts
+osm2pgsql-replication init \
+  -d spirit \
+  --server '<%= node[:vectortile][:replication][:url] %>'

diff --git a/cookbooks/vectortile/templates/default/index.html.erb b/cookbooks/vectortile/templates/default/index.html.erb
new file mode 100644 (file)
index 0000000..feb7f36
--- /dev/null
+++ b/cookbooks/vectortile/templates/default/index.html.erb
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<title>vector.openstreetmap.org</title>
+<!-- MANAGED VIA CHEF -->
+<meta name="robots" content="noindex">
+<meta http-equiv="cache-control" content="no-cache">
+<meta http-equiv="pragma" content="no-cache">
+</head>
+<body>
+You've reached the OpenStreetMap.org vector tile server (<%= node['fqdn'] %>)<br />
+<dl>
+<dt>If you are a user...</dt>
+<dd>You probably want <a href="https://www.openstreetmap.org/">OpenStreetMap</a> itself.</dd>
+<dt>If you are a developer...</dt>
+<dd>Please be aware of the <a href="https://operations.osmfoundation.org/policies/tiles/">tile usage policy</a>.</dd>
+</dl>
+</body>
+</html>

diff --git a/cookbooks/vectortile/templates/default/nginx.erb b/cookbooks/vectortile/templates/default/nginx.erb
new file mode 100644 (file)
index 0000000..838d30a
--- /dev/null
+++ b/cookbooks/vectortile/templates/default/nginx.erb
@@ -0,0 +1,46 @@
+upstream tilekiln {
+    server 127.0.0.1:8000;
+}
+
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    location /nginx_status {
+        stub_status on;
+        access_log   off;
+        allow 127.0.0.1;
+        allow ::1;
+        deny all;
+    }
+
+     rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent;
+
+     location / {
+         return 301 https://$host$request_uri;
+     }
+}
+
+server {
+    # IPv4
+    listen       443 ssl default_server;
+    # IPv6
+    listen       [::]:443 ssl default_server;
+    http2 on;
+    server_name  localhost;
+
+    ssl_certificate /etc/ssl/certs/<%= node[:fqdn] %>.pem;
+    ssl_certificate_key /etc/ssl/private/<%= node[:fqdn] %>.key;
+
+    location /nginx_status {
+        stub_status on;
+        access_log   off;
+        allow 127.0.0.1;
+        allow ::1;
+        deny all;
+    }
+    location /shortbread_v1/ {
+        proxy_pass http://tilekiln;
+    }
+    root /srv/vector.openstreetmap.org/html;
+}

diff --git a/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb
new file mode 100644 (file)
index 0000000..972d0e4
--- /dev/null
+++ b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Usage
+# sudo -u tileupdate tilekiln-storage-init
+
+<%= @tilekiln_bin %> storage init \
+  --storage-dbname <%= @storage_database %> \
+  --config <%= @config_path %>

diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb
new file mode 100644 (file)
index 0000000..f46b6f1
--- /dev/null
+++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb
@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+<%= @tilekiln_bin %> generate tiles \
+--source-dbname "<%= @source_database %>" \
+--storage-dbname "<%= @storage_database %>" \
+--num-threads "<%= node[:vectortile][:replication][:threads] %>" \
+--config <%= @config_path %> < <%= @tiles_file %>

diff --git a/cookbooks/vectortile/templates/default/vector-update-notile.erb b/cookbooks/vectortile/templates/default/vector-update-notile.erb
new file mode 100644 (file)
index 0000000..cd71878
--- /dev/null
+++ b/cookbooks/vectortile/templates/default/vector-update-notile.erb
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Usage
+# sudo -u tilekiln vector-update
+
+set -eu
+
+export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;'
+
+osm2pgsql-replication update \
+  -d "<%= @source_database %>" \
+  --max-diff-size "<%= @diff_size %>"

diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb
new file mode 100644 (file)
index 0000000..e59923b
--- /dev/null
+++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# Usage
+# sudo -u tilekiln vector-update
+
+set -eu
+
+export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;'
+
+osm2pgsql-replication update \
+  -d "<%= @source_database %>" \
+  --max-diff-size "<%= @diff_size %>" \
+  --post-processing "<%= @post_processing %>" \
+  -- --expire-tiles=10-14 \
+  --expire-output="<%= @tiles_file %>"

diff --git a/roles/vectortile.rb b/roles/vectortile.rb
new file mode 100644 (file)
index 0000000..189cddb
--- /dev/null
+++ b/roles/vectortile.rb
@@ -0,0 +1,72 @@
+name "vectortile"
+description "Role applied to all vector tile servers"
+
+default_attributes(
+  :accounts => {
+    :users => {
+      :pnorman => { :status => :administrator },
+      :tile => {
+        :members => [:tomh, :pnorman]
+      }
+    }
+  },
+  :postgresql => {
+    :settings => {
+      :defaults => {
+        :max_connections => "250",
+        :shared_buffers => "16GB",
+        :work_mem => "128MB",
+        :maintenance_work_mem => "8GB",
+        :max_parallel_workers_per_gather => "0",
+        :wal_level => "minimal",
+        :wal_buffers => "1024kB",
+        :wal_writer_delay => "500ms",
+        :checkpoint_timeout => "60min",
+        :commit_delay => "10000",
+        :max_wal_size => "10GB",
+        :max_wal_senders => "0",
+        :jit => "off",
+        :track_activity_query_size => "16384",
+        :autovacuum_vacuum_scale_factor => "0.05",
+        :autovacuum_analyze_scale_factor => "0.02"
+      }
+    }
+  },
+  :sysctl => {
+    :sockets => {
+      :comment => "Increase size of connection queue",
+      :parameters => {
+        "net.core.somaxconn" => 10000
+      }
+    },
+    :network_conntrack_time_wait => {
+      :comment => "Only track completed connections for 30 seconds",
+      :parameters => {
+        "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30"
+      }
+    },
+    :network_conntrack_max => {
+      :comment => "Increase max number of connections tracked",
+      :parameters => {
+        "net.netfilter.nf_conntrack_max" => "524288"
+      }
+    },
+    :no_tcp_slow_start => {
+      :comment => "Disable TCP slow start",
+      :parameters => {
+        "net.ipv4.tcp_slow_start_after_idle" => "0"
+      }
+    },
+    :tcp_use_bbr => {
+      :comment => "Use TCP BBR Congestion Control",
+      :parameters => {
+        "net.core.default_qdisc" => "fq",
+        "net.ipv4.tcp_congestion_control" => "bbr"
+      }
+    }
+  }
+)
+
+run_list(
+  "recipe[vectortile]"
+)

diff --git a/test/data_bags/accounts/tilekiln.json b/test/data_bags/accounts/tilekiln.json
new file mode 100644 (file)
index 0000000..100e017
--- /dev/null
+++ b/test/data_bags/accounts/tilekiln.json
@@ -0,0 +1,6 @@
+{
+  "id": "tilekiln",
+  "uid": "532",
+  "comment": "vectortile.openstreetmap.org",
+  "manage_home": false
+}

diff --git a/test/data_bags/accounts/tileupdate.json b/test/data_bags/accounts/tileupdate.json
new file mode 100644 (file)
index 0000000..8b6ba87
--- /dev/null
+++ b/test/data_bags/accounts/tileupdate.json
@@ -0,0 +1,6 @@
+{
+  "id": "tileupdate",
+  "uid": "531",
+  "comment": "vectortile.openstreetmap.org",
+  "manage_home": false
+}

diff --git a/test/integration/vectortile/inspec/import_spec.rb b/test/integration/vectortile/inspec/import_spec.rb
new file mode 100644 (file)
index 0000000..13ff57e
--- /dev/null
+++ b/test/integration/vectortile/inspec/import_spec.rb
@@ -0,0 +1,3 @@
+describe file("/usr/local/bin/import-planet") do
+  it { should be_executable.by_user("tileupdate") }
+end

diff --git a/test/integration/vectortile/inspec/nginx_spec.rb b/test/integration/vectortile/inspec/nginx_spec.rb
new file mode 100644 (file)
index 0000000..75c1e16
--- /dev/null
+++ b/test/integration/vectortile/inspec/nginx_spec.rb
@@ -0,0 +1,26 @@
+describe package("nginx") do
+  it { should be_installed }
+end
+
+describe service("nginx") do
+  it { should be_enabled }
+  it { should be_running }
+end
+
+describe port(80) do
+  it { should be_listening }
+  its("protocols") { should cmp %w[tcp tcp6] }
+end
+
+describe port(443) do
+  it { should be_listening }
+  its("protocols") { should cmp %w[tcp tcp6] }
+end
+
+describe http("http://localhost") do
+  its("status") { should cmp 301 }
+end
+
+describe http("https://localhost", :ssl_verify => false) do
+  its("status") { should cmp 200 }
+end

diff --git a/test/integration/vectortile/inspec/osm2pgsql_spec.rb b/test/integration/vectortile/inspec/osm2pgsql_spec.rb
new file mode 100644 (file)
index 0000000..e5efe33
--- /dev/null
+++ b/test/integration/vectortile/inspec/osm2pgsql_spec.rb
@@ -0,0 +1,4 @@
+describe package("osm2pgsql") do
+  it { should be_installed }
+  its("version") { should cmp >= "1.11.0" }
+end

diff --git a/test/integration/vectortile/inspec/postgresql_spec.rb b/test/integration/vectortile/inspec/postgresql_spec.rb
new file mode 100644 (file)
index 0000000..925d674
--- /dev/null
+++ b/test/integration/vectortile/inspec/postgresql_spec.rb
@@ -0,0 +1,13 @@
+describe package("postgresql-16") do
+  it { should be_installed }
+end
+
+describe service("postgresql@16-main") do
+  it { should be_enabled }
+  it { should be_running }
+end
+
+describe port(5432) do
+  it { should be_listening }
+  its("protocols") { should cmp %w[tcp tcp6] }
+end

diff --git a/test/integration/vectortile/inspec/storage_spec.rb b/test/integration/vectortile/inspec/storage_spec.rb
new file mode 100644 (file)
index 0000000..9498695
--- /dev/null
+++ b/test/integration/vectortile/inspec/storage_spec.rb
@@ -0,0 +1,3 @@
+describe file("/usr/local/bin/tilekiln-storage-init") do
+  it { should be_executable.by_user("tileupdate") }
+end

diff --git a/test/integration/vectortile/inspec/themepark_spec.rb b/test/integration/vectortile/inspec/themepark_spec.rb
new file mode 100644 (file)
index 0000000..eb7a970
--- /dev/null
+++ b/test/integration/vectortile/inspec/themepark_spec.rb
@@ -0,0 +1,7 @@
+describe file("/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/themepark.lua") do
+  it { should exist }
+end
+
+describe file("/srv/vector.openstreetmap.org/spirit/spirit.lua") do
+  it { should exist }
+end

diff --git a/test/integration/vectortile/inspec/tilekiln_spec.rb b/test/integration/vectortile/inspec/tilekiln_spec.rb
new file mode 100644 (file)
index 0000000..2887571
--- /dev/null
+++ b/test/integration/vectortile/inspec/tilekiln_spec.rb
@@ -0,0 +1,30 @@
+describe pip("tilekiln", "/opt/tilekiln/bin/pip") do
+  it { should be_installed }
+  its("version") { should cmp >= "0.5.0" }
+end
+
+describe service("tilekiln") do
+  it { should be_enabled }
+  it { should be_running }
+end
+
+describe port(8000) do
+  it { should be_listening }
+  its("protocols") { should cmp %w[tcp] }
+end
+
+describe http("http://localhost:8000") do
+  its("status") { should cmp 404 }
+end
+
+describe http("https://localhost/shortbread_v1/tilejson.json", :ssl_verify => false) do
+  its("status") { should cmp 200 }
+end
+
+# There are no tiles so everything should return a 404
+describe http("https://localhost/shortbread_v1/0/0/0.mvt", :ssl_verify => false) do
+  its("status") { should cmp 404 }
+end
+describe http("https://localhost/shortbread_v1/16/0/0.mvt", :ssl_verify => false) do
+  its("status") { should cmp 404 }
+end