From: Paul Norman Date: Thu, 4 Jul 2024 00:03:00 +0000 (-0700) Subject: Create a new vectortile role X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/04743763991d4bd0b536ceaec43439d6b3b4842c Create a new vectortile role This role is for tilekiln based vector tile serving --- diff --git a/.kitchen.yml b/.kitchen.yml index 0af8c80de..f7cfa5d38 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -416,6 +416,9 @@ suites: - name: trac run_list: - recipe[trac::default] + - name: vectortile + run_list: + - recipe[vectortile::default] - name: web-cgimap run_list: - recipe[web::cgimap] diff --git a/cookbooks/postgresql/libraries/postgresql.rb b/cookbooks/postgresql/libraries/postgresql.rb index 1a39da25a..b2df4aed6 100644 --- a/cookbooks/postgresql/libraries/postgresql.rb +++ b/cookbooks/postgresql/libraries/postgresql.rb @@ -4,6 +4,10 @@ module OpenStreetMap class PostgreSQL include Chef::Mixin::ShellOut + SCHEMA_PRIVILEGES = [ + :create, :usage + ].freeze + TABLE_PRIVILEGES = [ :select, :insert, :update, :delete, :truncate, :references, :trigger ].freeze @@ -115,9 +119,21 @@ module OpenStreetMap end end + def schemas(database) + @schemas ||= {} + @schemas[database] ||= query("SELECT n.nspname, pg_catalog.pg_get_userbyid(n.nspowner) AS usename, n.nspacl FROM pg_namespace AS n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema'", :database => database).each_with_object({}) do |schema, schemas| + name = "#{schema[:nspname]}" + + schemas[name] = { + :owner => schema[:usename], + :permissions => parse_acl(schema[:nspacl] || "{}") + } + end + end + def tables(database) @tables ||= {} - @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind = 'r'", :database => database).each_with_object({}) do |table, tables| + @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind IN ('r', 'p')", :database => database).each_with_object({}) do |table, tables| name = "#{table[:nspname]}.#{table[:relname]}" tables[name] = { diff --git a/cookbooks/postgresql/resources/schema.rb b/cookbooks/postgresql/resources/schema.rb new file mode 100644 index 000000000..a7bf0ebdb --- /dev/null +++ b/cookbooks/postgresql/resources/schema.rb @@ -0,0 +1,114 @@ +# +# Cookbook:: postgresql +# Resource:: postgresql_schema +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +unified_mode true + +default_action :create + +property :schema, :kind_of => String, :name_property => true +property :cluster, :kind_of => String, :required => true +property :database, :kind_of => String, :required => true +property :owner, :kind_of => String, :required => [:create] +property :permissions, :kind_of => Hash, :default => {} + +action :create do + if schemas.include?(qualified_name) + # Handle the case of an existing schema + if new_resource.owner != schemas[qualified_name][:owner] + converge_by("set owner for #{new_resource} to #{new_resource.owner}") do + Chef::Log.info("Setting owner for #{new_resource} to #{new_resource.owner}") + cluster.execute(:command => "ALTER SCHEMA #{qualified_name} OWNER TO \"#{new_resource.owner}\"", :database => new_resource.database) + end + end + + schemas[qualified_name][:permissions].each_key do |user| + next if new_resource.permissions[user] + # Remove permissions from users who no longer have them + converge_by("revoke all for #{user} on #{new_resource}") do + Chef::Log.info("Revoking all for #{user} on #{new_resource}") + cluster.execute(:command => "REVOKE ALL ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database) + end + end + new_resource.permissions.each do |user, new_privileges| + current_privileges = schemas[qualified_name][:permissions][user] || {} + new_privileges = Array(new_privileges) + + if new_privileges.include?(:all) + new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES + end + + OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege| + if new_privileges.include?(privilege) + unless current_privileges.include?(privilege) + converge_by("grant #{privilege} for #{user} on #{new_resource}") do + Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}") + cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database) + end + end + elsif current_privileges.include?(privilege) + converge_by("revoke #{privilege} for #{user} on #{new_resource}") do + Chef::Log.info("Revoking #{privilege} for #{user} on #{new_resource}") + cluster.execute(:command => "REVOKE #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database) + end + end + end + end + else + converge_by "create schema #{new_resource.schema}" do + cluster.execute(:command => "CREATE SCHEMA #{new_resource.schema} AUTHORIZATION #{new_resource.owner}", :database => new_resource.database) + # Because the schema is new, we don't have to worry about revoking or checking current permissions + new_resource.permissions.each do |user, new_privileges| + new_privileges = Array(new_privileges) + if new_privileges.include?(:all) + new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES + end + OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege| + next unless new_privileges.include?(privilege) + converge_by("grant #{privilege} for #{user} on #{new_resource}") do + Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}") + cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database) + end + end + end + end + end +end + +action :drop do + if schemas.include?(qualified_name) + converge_by("drop #{new_resource}") do + Chef::Log.info("Dropping #{new_resource}") + cluster.execute(:command => "DROP SCHEMA #{qualified_name}", :database => new_resource.database) + end + end +end + +action_class do + def cluster + @cluster ||= OpenStreetMap::PostgreSQL.new(new_resource.cluster) + end + + def schemas + @schemas ||= cluster.schemas(new_resource.database) + end + + def qualified_name + "#{new_resource.name}" + end +end diff --git a/cookbooks/vectortile/README.md b/cookbooks/vectortile/README.md new file mode 100644 index 000000000..659ee7aa4 --- /dev/null +++ b/cookbooks/vectortile/README.md @@ -0,0 +1,9 @@ +# vectortile cookbook + +This cookbook installs and configures the tilekiln based tileservers + +## Accounts +The following accounts are used +- `www-data` for nginx serving static files and proxying +- `tilekiln` for the process serving tiles +- `tileupdate` for the process running osm2pgsql and tilekiln on updates diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb new file mode 100644 index 000000000..0df4249e7 --- /dev/null +++ b/cookbooks/vectortile/attributes/default.rb @@ -0,0 +1,15 @@ +default[:vectortile][:database][:cluster] = "16/main" +default[:vectortile][:database][:postgis] = "3" +default[:vectortile][:database][:nodes_store] = :flat +default[:vectortile][:serve][:threads] = node.cpu_cores +default[:vectortile][:serve][:mode] = :live +default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute" +default[:vectortile][:replication][:status] = :enabled +default[:vectortile][:replication][:tileupdate] = :enabled +default[:vectortile][:replication][:threads] = node.cpu_cores + +default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] +default[:postgresql][:monitor_database] = "tiles" +default[:postgresql][:settings][:defaults][:max_connections] = (node.cpu_cores * 4 + 20).to_s +default[:accounts][:users][:tileupdate][:status] = :role +default[:accounts][:users][:tilekiln][:status] = :role diff --git a/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml b/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml new file mode 100644 index 000000000..d60ec1d3f --- /dev/null +++ b/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/cookbooks/vectortile/files/default/html/crossdomain.xml b/cookbooks/vectortile/files/default/html/crossdomain.xml new file mode 100644 index 000000000..35fc43447 --- /dev/null +++ b/cookbooks/vectortile/files/default/html/crossdomain.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/cookbooks/vectortile/files/default/html/favicon.ico b/cookbooks/vectortile/files/default/html/favicon.ico new file mode 100644 index 000000000..975e1cb0d Binary files /dev/null and b/cookbooks/vectortile/files/default/html/favicon.ico differ diff --git a/cookbooks/vectortile/files/default/html/robots.txt b/cookbooks/vectortile/files/default/html/robots.txt new file mode 100644 index 000000000..bfb50b0ae --- /dev/null +++ b/cookbooks/vectortile/files/default/html/robots.txt @@ -0,0 +1,13 @@ +User-agent: * +Disallow: /12/ +Disallow: /13/ +Disallow: /14/ +Disallow: /15/ +Disallow: /16/ +Disallow: /17/ +Disallow: /18/ +Disallow: /19/ +Disallow: /20/ +Disallow: /21/ +Disallow: /22/ +Disallow: /23/ diff --git a/cookbooks/vectortile/metadata.rb b/cookbooks/vectortile/metadata.rb new file mode 100644 index 000000000..d3b44e37e --- /dev/null +++ b/cookbooks/vectortile/metadata.rb @@ -0,0 +1,16 @@ +name "vectortile" +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache-2.0" +description "Installs and configures vector tile servers" + +version "1.0.0" +supports "ubuntu" +depends "accounts" +depends "git" +depends "nginx" +depends "postgresql" +depends "prometheus" +depends "python" +depends "systemd" +depends "tools" diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb new file mode 100644 index 000000000..37fe86b43 --- /dev/null +++ b/cookbooks/vectortile/recipes/default.rb @@ -0,0 +1,282 @@ +# +# Cookbook:: vectortile +# Recipe:: default +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "accounts" +include_recipe "git" +include_recipe "nginx" +include_recipe "postgresql" +include_recipe "prometheus" +include_recipe "python" +include_recipe "tools" + +directory "/srv/vector.openstreetmap.org" do + user "tileupdate" + group "tileupdate" + mode "755" +end + +nginx_site "default" do + action [:delete] +end + +nginx_site "vector.openstreetmap.org" do + template "nginx.erb" +end + +ssl_certificate node[:fqdn] do + domains [node[:fqdn], "vector.openstreetmap.org"] + notifies :reload, "service[nginx]" +end + +remote_directory "/srv/vector.openstreetmap.org/html" do + source "html" + owner "www-data" + group "www-data" + mode "755" + files_owner "www-data" + files_group "www-data" + files_mode "644" +end + +template "/srv/vector.openstreetmap.org/html/index.html" do + source "index.html.erb" + owner "www-data" + group "www-data" + mode "644" +end + +postgresql_version = node[:vectortile][:database][:cluster].split("/").first +postgis_version = node[:vectortile][:database][:postgis] + +package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" + +# Spirit dependencies +package %w[ + osm2pgsql + gdal-bin + python3-yaml + python3-psycopg2 +] + +style_directory = "/srv/vector.openstreetmap.org/spirit" +git style_directory do + repository "https://github.com/pnorman/spirit.git" + user "tileupdate" + group "tileupdate" +end + +shortbread_config = "#{style_directory}/shortbread.yaml" + +themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark" +git themepark_directory do + repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git" + user "tileupdate" + group "tileupdate" +end + +tilekiln_directory = "/opt/tilekiln" + +python_virtualenv tilekiln_directory do + interpreter "/usr/bin/python3" +end + +python_package "tilekiln" do + python_virtualenv tilekiln_directory + python_version "3" + version "0.5.1" +end + +template "/srv/vector.openstreetmap.org/html/index.html" do + source "index.html.erb" + owner "www-data" + group "www-data" + mode "644" +end + +directory "/srv/vector.openstreetmap.org/data" do + user "tileupdate" + group "tileupdate" + mode "755" +end + +node_store_options = node[:vectortile][:database][:nodes_store] == :flat ? "--flat-nodes '/srv/vector.openstreetmap.org/data/nodes.bin'" : "" +template "/usr/local/bin/import-planet" do + source "import-planet.erb" + owner "root" + group "root" + mode "755" + variables :node_store_options => "#{node_store_options}" +end + +template "/usr/local/bin/tilekiln-storage-init" do + source "tilekiln-storage-init.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{shortbread_config}" +end + +postgresql_user "tomh" do + cluster node[:vectortile][:database][:cluster] + superuser true +end + +postgresql_user "pnorman" do + cluster node[:vectortile][:database][:cluster] + superuser true +end + +postgresql_user "tilekiln" do + cluster node[:vectortile][:database][:cluster] +end + +postgresql_user "tileupdate" do + cluster node[:vectortile][:database][:cluster] +end + +postgresql_database "tiles" do + cluster node[:vectortile][:database][:cluster] + owner "tileupdate" +end + +postgresql_database "spirit" do + cluster node[:vectortile][:database][:cluster] + owner "tileupdate" +end + +postgresql_extension "postgis" do + cluster node[:vectortile][:database][:cluster] + database "spirit" +end + +postgresql_schema "tilekiln" do + cluster node[:vectortile][:database][:cluster] + database "tiles" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => :usage + notifies :run, "execute[tilekiln-storage-init]", :immediately +end + +execute "tilekiln-storage-init" do + action :nothing + command "/usr/local/bin/tilekiln-storage-init" + user "tileupdate" +end + +%w[metadata shortbread_v1].each do |table| + postgresql_table table do + cluster node[:vectortile][:database][:cluster] + database "tiles" + schema "tilekiln" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => :select + end +end + +(0..14).each do |zoom| + postgresql_table "shortbread_v1_z#{zoom}" do + cluster node[:vectortile][:database][:cluster] + database "tiles" + schema "tilekiln" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => node[:vectortile][:serve][:mode] == :live ? [:select, :insert, :update] : :select + end +end + +%w[addresses aerialways aeroways boundaries boundary_labels bridges buildings +dam_lines dam_polygons ferries land pier_lines pier_polygons place_labels +planet_osm_nodes planet_osm_rels planet_osm_ways pois public_transport railways +road_routes roads sites street_polygons streets_labels_points +streets_polygons_labels water_area_labels water_areas water_lines water_lines_labels].each do |table| + postgresql_table table do + cluster node[:vectortile][:database][:cluster] + database "spirit" + schema "public" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => :select + end +end + +tilekiln_mode = node[:vectortile][:serve][:mode] == :live ? "live --config #{shortbread_config} --source-dbname spirit" : "static" + +systemd_service "tilekiln" do + description "Tilekiln vector tile server" + user "tilekiln" + after "postgresql.service" + wants "postgresql.service" + sandbox :enable_network => true + restrict_address_families "AF_UNIX" + exec_start "#{tilekiln_directory}/bin/tilekiln serve #{tilekiln_mode} --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]}" +end + +service "tilekiln" do + action [:enable, :start] +end + +execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do + command "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py -R tilekiln" + cwd "/srv/vector.openstreetmap.org/spirit" + user "tileupdate" + group "tileupdate" + ignore_failure true +end + +template "/usr/local/bin/vector-update" do + source node[:vectortile][:replication][:tileupdate] == :enabled ? "vector-update-tile.erb" : "vector-update-notile.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :post_processing => "/usr/local/bin/tiles-rerender" +end + +template "/usr/local/bin/tiles-rerender" do + source "tiles-rerender.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :update_threads => 4 +end + +systemd_service "replicate" do + description "Get replication updates" + user "tileupdate" + after "postgresql.service" + wants "postgresql.service" + sandbox :enable_network => true + restrict_address_families "AF_UNIX" + read_write_paths ["/srv/vector.openstreetmap.org/data/"] + exec_start "/usr/local/bin/vector-update" +end + +systemd_timer "replicate" do + description "Get replication updates" + on_boot_sec 60 + on_unit_active_sec 30 + accuracy_sec 5 +end + +if node[:vectortile][:replication][:status] == :enabled + service "replicate.timer" do + action [:enable, :start] + end +else + service "replicate.timer" do + action [:stop, :disable] + end +end diff --git a/cookbooks/vectortile/templates/default/import-planet.erb b/cookbooks/vectortile/templates/default/import-planet.erb new file mode 100644 index 000000000..bffa34a3a --- /dev/null +++ b/cookbooks/vectortile/templates/default/import-planet.erb @@ -0,0 +1,24 @@ +#!/bin/sh + +# Usage +# sudo -u tileupdate import-planet osmfile.osm.pbf +# sudo -u tileupdate import-planet --cache 50000 osmfile.osm.pbf +# This script sets the appropriate Lua and style paths for the osm2pgsql import + +set -e + +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' + +# Import the osm2pgsql file specified as an argument, using the locations for spirit +osm2pgsql \ + --output flex \ + --style '/srv/vector.openstreetmap.org/spirit/shortbread.lua' \ + --slim \ + <%= @node_store_options %> \ + -d spirit \ + $@ + +# Set up replication. This doesn't specify the replication server, so it will use planet.osm.org on extracts +osm2pgsql-replication init \ + -d spirit \ + --server '<%= node[:vectortile][:replication][:url] %>' diff --git a/cookbooks/vectortile/templates/default/index.html.erb b/cookbooks/vectortile/templates/default/index.html.erb new file mode 100644 index 000000000..feb7f36fa --- /dev/null +++ b/cookbooks/vectortile/templates/default/index.html.erb @@ -0,0 +1,19 @@ + + + +vector.openstreetmap.org + + + + + + +You've reached the OpenStreetMap.org vector tile server (<%= node['fqdn'] %>)
+
+
If you are a user...
+
You probably want OpenStreetMap itself.
+
If you are a developer...
+
Please be aware of the tile usage policy.
+
+ + diff --git a/cookbooks/vectortile/templates/default/nginx.erb b/cookbooks/vectortile/templates/default/nginx.erb new file mode 100644 index 000000000..838d30a4f --- /dev/null +++ b/cookbooks/vectortile/templates/default/nginx.erb @@ -0,0 +1,46 @@ +upstream tilekiln { + server 127.0.0.1:8000; +} + +server { + listen 80 default_server; + listen [::]:80 default_server; + + location /nginx_status { + stub_status on; + access_log off; + allow 127.0.0.1; + allow ::1; + deny all; + } + + rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; + + location / { + return 301 https://$host$request_uri; + } +} + +server { + # IPv4 + listen 443 ssl default_server; + # IPv6 + listen [::]:443 ssl default_server; + http2 on; + server_name localhost; + + ssl_certificate /etc/ssl/certs/<%= node[:fqdn] %>.pem; + ssl_certificate_key /etc/ssl/private/<%= node[:fqdn] %>.key; + + location /nginx_status { + stub_status on; + access_log off; + allow 127.0.0.1; + allow ::1; + deny all; + } + location /shortbread_v1/ { + proxy_pass http://tilekiln; + } + root /srv/vector.openstreetmap.org/html; +} diff --git a/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb new file mode 100644 index 000000000..972d0e488 --- /dev/null +++ b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb @@ -0,0 +1,8 @@ +#!/bin/sh + +# Usage +# sudo -u tileupdate tilekiln-storage-init + +<%= @tilekiln_bin %> storage init \ + --storage-dbname <%= @storage_database %> \ + --config <%= @config_path %> diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb new file mode 100644 index 000000000..f46b6f1f6 --- /dev/null +++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +<%= @tilekiln_bin %> generate tiles \ +--source-dbname "<%= @source_database %>" \ +--storage-dbname "<%= @storage_database %>" \ +--num-threads "<%= node[:vectortile][:replication][:threads] %>" \ +--config <%= @config_path %> < <%= @tiles_file %> diff --git a/cookbooks/vectortile/templates/default/vector-update-notile.erb b/cookbooks/vectortile/templates/default/vector-update-notile.erb new file mode 100644 index 000000000..cd718789c --- /dev/null +++ b/cookbooks/vectortile/templates/default/vector-update-notile.erb @@ -0,0 +1,12 @@ +#!/bin/sh + +# Usage +# sudo -u tilekiln vector-update + +set -eu + +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' + +osm2pgsql-replication update \ + -d "<%= @source_database %>" \ + --max-diff-size "<%= @diff_size %>" diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb new file mode 100644 index 000000000..e59923b27 --- /dev/null +++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb @@ -0,0 +1,15 @@ +#!/bin/sh + +# Usage +# sudo -u tilekiln vector-update + +set -eu + +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' + +osm2pgsql-replication update \ + -d "<%= @source_database %>" \ + --max-diff-size "<%= @diff_size %>" \ + --post-processing "<%= @post_processing %>" \ + -- --expire-tiles=10-14 \ + --expire-output="<%= @tiles_file %>" diff --git a/roles/vectortile.rb b/roles/vectortile.rb new file mode 100644 index 000000000..189cddb8d --- /dev/null +++ b/roles/vectortile.rb @@ -0,0 +1,72 @@ +name "vectortile" +description "Role applied to all vector tile servers" + +default_attributes( + :accounts => { + :users => { + :pnorman => { :status => :administrator }, + :tile => { + :members => [:tomh, :pnorman] + } + } + }, + :postgresql => { + :settings => { + :defaults => { + :max_connections => "250", + :shared_buffers => "16GB", + :work_mem => "128MB", + :maintenance_work_mem => "8GB", + :max_parallel_workers_per_gather => "0", + :wal_level => "minimal", + :wal_buffers => "1024kB", + :wal_writer_delay => "500ms", + :checkpoint_timeout => "60min", + :commit_delay => "10000", + :max_wal_size => "10GB", + :max_wal_senders => "0", + :jit => "off", + :track_activity_query_size => "16384", + :autovacuum_vacuum_scale_factor => "0.05", + :autovacuum_analyze_scale_factor => "0.02" + } + } + }, + :sysctl => { + :sockets => { + :comment => "Increase size of connection queue", + :parameters => { + "net.core.somaxconn" => 10000 + } + }, + :network_conntrack_time_wait => { + :comment => "Only track completed connections for 30 seconds", + :parameters => { + "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30" + } + }, + :network_conntrack_max => { + :comment => "Increase max number of connections tracked", + :parameters => { + "net.netfilter.nf_conntrack_max" => "524288" + } + }, + :no_tcp_slow_start => { + :comment => "Disable TCP slow start", + :parameters => { + "net.ipv4.tcp_slow_start_after_idle" => "0" + } + }, + :tcp_use_bbr => { + :comment => "Use TCP BBR Congestion Control", + :parameters => { + "net.core.default_qdisc" => "fq", + "net.ipv4.tcp_congestion_control" => "bbr" + } + } + } +) + +run_list( + "recipe[vectortile]" +) diff --git a/test/data_bags/accounts/tilekiln.json b/test/data_bags/accounts/tilekiln.json new file mode 100644 index 000000000..100e01771 --- /dev/null +++ b/test/data_bags/accounts/tilekiln.json @@ -0,0 +1,6 @@ +{ + "id": "tilekiln", + "uid": "532", + "comment": "vectortile.openstreetmap.org", + "manage_home": false +} diff --git a/test/data_bags/accounts/tileupdate.json b/test/data_bags/accounts/tileupdate.json new file mode 100644 index 000000000..8b6ba87c6 --- /dev/null +++ b/test/data_bags/accounts/tileupdate.json @@ -0,0 +1,6 @@ +{ + "id": "tileupdate", + "uid": "531", + "comment": "vectortile.openstreetmap.org", + "manage_home": false +} diff --git a/test/integration/vectortile/inspec/import_spec.rb b/test/integration/vectortile/inspec/import_spec.rb new file mode 100644 index 000000000..13ff57e87 --- /dev/null +++ b/test/integration/vectortile/inspec/import_spec.rb @@ -0,0 +1,3 @@ +describe file("/usr/local/bin/import-planet") do + it { should be_executable.by_user("tileupdate") } +end diff --git a/test/integration/vectortile/inspec/nginx_spec.rb b/test/integration/vectortile/inspec/nginx_spec.rb new file mode 100644 index 000000000..75c1e16b8 --- /dev/null +++ b/test/integration/vectortile/inspec/nginx_spec.rb @@ -0,0 +1,26 @@ +describe package("nginx") do + it { should be_installed } +end + +describe service("nginx") do + it { should be_enabled } + it { should be_running } +end + +describe port(80) do + it { should be_listening } + its("protocols") { should cmp %w[tcp tcp6] } +end + +describe port(443) do + it { should be_listening } + its("protocols") { should cmp %w[tcp tcp6] } +end + +describe http("http://localhost") do + its("status") { should cmp 301 } +end + +describe http("https://localhost", :ssl_verify => false) do + its("status") { should cmp 200 } +end diff --git a/test/integration/vectortile/inspec/osm2pgsql_spec.rb b/test/integration/vectortile/inspec/osm2pgsql_spec.rb new file mode 100644 index 000000000..e5efe3340 --- /dev/null +++ b/test/integration/vectortile/inspec/osm2pgsql_spec.rb @@ -0,0 +1,4 @@ +describe package("osm2pgsql") do + it { should be_installed } + its("version") { should cmp >= "1.11.0" } +end diff --git a/test/integration/vectortile/inspec/postgresql_spec.rb b/test/integration/vectortile/inspec/postgresql_spec.rb new file mode 100644 index 000000000..925d6746b --- /dev/null +++ b/test/integration/vectortile/inspec/postgresql_spec.rb @@ -0,0 +1,13 @@ +describe package("postgresql-16") do + it { should be_installed } +end + +describe service("postgresql@16-main") do + it { should be_enabled } + it { should be_running } +end + +describe port(5432) do + it { should be_listening } + its("protocols") { should cmp %w[tcp tcp6] } +end diff --git a/test/integration/vectortile/inspec/storage_spec.rb b/test/integration/vectortile/inspec/storage_spec.rb new file mode 100644 index 000000000..9498695d1 --- /dev/null +++ b/test/integration/vectortile/inspec/storage_spec.rb @@ -0,0 +1,3 @@ +describe file("/usr/local/bin/tilekiln-storage-init") do + it { should be_executable.by_user("tileupdate") } +end diff --git a/test/integration/vectortile/inspec/themepark_spec.rb b/test/integration/vectortile/inspec/themepark_spec.rb new file mode 100644 index 000000000..eb7a97013 --- /dev/null +++ b/test/integration/vectortile/inspec/themepark_spec.rb @@ -0,0 +1,7 @@ +describe file("/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/themepark.lua") do + it { should exist } +end + +describe file("/srv/vector.openstreetmap.org/spirit/spirit.lua") do + it { should exist } +end diff --git a/test/integration/vectortile/inspec/tilekiln_spec.rb b/test/integration/vectortile/inspec/tilekiln_spec.rb new file mode 100644 index 000000000..2887571db --- /dev/null +++ b/test/integration/vectortile/inspec/tilekiln_spec.rb @@ -0,0 +1,30 @@ +describe pip("tilekiln", "/opt/tilekiln/bin/pip") do + it { should be_installed } + its("version") { should cmp >= "0.5.0" } +end + +describe service("tilekiln") do + it { should be_enabled } + it { should be_running } +end + +describe port(8000) do + it { should be_listening } + its("protocols") { should cmp %w[tcp] } +end + +describe http("http://localhost:8000") do + its("status") { should cmp 404 } +end + +describe http("https://localhost/shortbread_v1/tilejson.json", :ssl_verify => false) do + its("status") { should cmp 200 } +end + +# There are no tiles so everything should return a 404 +describe http("https://localhost/shortbread_v1/0/0/0.mvt", :ssl_verify => false) do + its("status") { should cmp 404 } +end +describe http("https://localhost/shortbread_v1/16/0/0.mvt", :ssl_verify => false) do + its("status") { should cmp 404 } +end