X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/93e92b84438ecf5422bdd9c685ae986be229068d..e0561daeb2ceeb25a6be1c228f4bd0901197f53a:/cookbooks/tile/recipes/default.rb diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 74f011cc1..f642aafb8 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -1,14 +1,14 @@ # -# Cookbook Name:: tile +# Cookbook:: tile # Recipe:: default # -# Copyright 2013, OpenStreetMap Foundation +# Copyright:: 2013, OpenStreetMap Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -17,14 +17,19 @@ # limitations under the License. # +include_recipe "accounts" include_recipe "apache" include_recipe "git" +include_recipe "munin" include_recipe "nodejs" include_recipe "postgresql" +include_recipe "prometheus" include_recipe "python" +include_recipe "ruby" include_recipe "tools" blocks = data_bag_item("tile", "blocks") +admins = data_bag_item("apache", "admins") web_passwords = data_bag_item("web", "passwords") apache_module "alias" @@ -38,51 +43,73 @@ apache_module "tile" do conf "tile.conf.erb" end +apache_conf "renderd" do + action :disable +end + ssl_certificate node[:fqdn] do - domains [node[:fqdn], "render.openstreetmap.org"] + domains [node[:fqdn], "tile.openstreetmap.org", "render.openstreetmap.org"] notifies :reload, "service[apache2]" end -tilecaches = search(:node, "roles:tilecache").sort_by { |n| n[:hostname] } +remote_file "#{Chef::Config[:file_cache_path]}/fastly-ip-list.json" do + source "https://api.fastly.com/public-ip-list" + compile_time true + ignore_failure true +end + +fastlyips = JSON.parse(IO.read("#{Chef::Config[:file_cache_path]}/fastly-ip-list.json")) + +remote_file "#{Chef::Config[:file_cache_path]}/statuscake-locations.json" do + source "https://app.statuscake.com/Workfloor/Locations.php?format=json" + compile_time true + ignore_failure true +end + +statuscakelocations = JSON.parse(IO.read("#{Chef::Config[:file_cache_path]}/statuscake-locations.json")) apache_site "default" do - action [:disable] + action :disable +end + +apache_site "tileserver_site" do + action :disable end apache_site "tile.openstreetmap.org" do template "apache.erb" - variables :caches => tilecaches + variables :fastly => fastlyips["addresses"] + fastlyips["ipv6_addresses"], + :statuscake => statuscakelocations.flat_map { |_, v| [v["ip"], v["ipv6"]] }, + :admins => admins["hosts"] end template "/etc/logrotate.d/apache2" do source "logrotate.apache.erb" owner "root" group "root" - mode 0o644 + mode "644" end directory "/srv/tile.openstreetmap.org" do owner "tile" group "tile" - mode 0o755 + mode "755" end +tile_directories = node[:tile][:styles].collect do |_, style| + style[:tile_directories].collect { |directory| directory[:name] } +end.flatten.sort.uniq + package "renderd" systemd_service "renderd" do - description "Mapnik rendering daemon" + dropin "chef" after "postgresql.service" wants "postgresql.service" - user "www-data" - exec_start "/usr/bin/renderd -f" - runtime_directory "renderd" - standard_error "null" - private_tmp true - private_devices true - private_network true - protect_system "full" - protect_home true - no_new_privileges true + limit_nofile 4096 + sandbox true + restrict_address_families "AF_UNIX" + read_write_paths tile_directories restart "on-failure" end @@ -94,14 +121,14 @@ end directory "/srv/tile.openstreetmap.org/tiles" do owner "tile" group "tile" - mode 0o755 + mode "755" end template "/etc/renderd.conf" do source "renderd.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :reload, "service[apache2]" notifies :restart, "service[renderd]" end @@ -110,56 +137,41 @@ remote_directory "/srv/tile.openstreetmap.org/html" do source "html" owner "tile" group "tile" - mode 0o755 + mode "755" files_owner "tile" files_group "tile" - files_mode 0o644 + files_mode "644" end template "/srv/tile.openstreetmap.org/html/index.html" do source "index.html.erb" owner "tile" group "tile" - mode 0o644 + mode "644" end package %w[ - python-cairo - python-mapnik - python-setuptools + python3-cairo + python3-mapnik + python3-pyproj + python3-setuptools ] -python_package "pyotp" - -package %w[ - fonts-noto-cjk - fonts-noto-hinted - fonts-noto-unhinted - fonts-hanazono - ttf-unifont -] - -["NotoSansArabicUI-Regular.ttf", "NotoSansArabicUI-Bold.ttf"].each do |font| - remote_file "/usr/share/fonts/truetype/noto/#{font}" do - action :create_if_missing - source "https://github.com/googlei18n/noto-fonts/raw/master/hinted/#{font}" - owner "root" - group "root" - mode 0o644 - end +python_package "pyotp" do + python_version "3" end directory "/srv/tile.openstreetmap.org/cgi-bin" do owner "tile" group "tile" - mode 0o755 + mode "755" end template "/srv/tile.openstreetmap.org/cgi-bin/export" do source "export.erb" owner "tile" group "tile" - mode 0o755 + mode "755" variables :blocks => blocks, :totp_key => web_passwords["totp_key"] end @@ -167,23 +179,38 @@ template "/srv/tile.openstreetmap.org/cgi-bin/debug" do source "debug.erb" owner "tile" group "tile" - mode 0o755 + mode "755" end -template "/etc/cron.hourly/export" do - source "export.cron.erb" - owner "root" - group "root" - mode 0o755 +systemd_service "export-cleanup" do + description "Cleanup stale export temporary files" + joins_namespace_of "apache2.service" + exec_start "find /tmp -ignore_readdir_race -name 'export??????' -mmin +60 -delete" + user "www-data" + sandbox true +end + +systemd_timer "export-cleanup" do + description "Cleanup stale export temporary files" + on_boot_sec "60m" + on_unit_inactive_sec "60m" +end + +service "export-cleanup.timer" do + action [:enable, :start] end directory "/srv/tile.openstreetmap.org/data" do owner "tile" group "tile" - mode 0o755 + mode "755" end -package "mapnik-utils" +package %w[ + mapnik-utils + tar + unzip +] node[:tile][:data].each_value do |data| url = data[:url] @@ -195,15 +222,13 @@ node[:tile][:data].each_value do |data| directory directory do owner "tile" group "tile" - mode 0o755 + mode "755" end else directory = "/srv/tile.openstreetmap.org/data" end if file =~ /\.tgz$/ - package "tar" - execute file do action :nothing command "tar -zxf #{file} -C #{directory}" @@ -211,8 +236,6 @@ node[:tile][:data].each_value do |data| group "tile" end elsif file =~ /\.tar\.bz2$/ - package "tar" - execute file do action :nothing command "tar -jxf #{file} -C #{directory}" @@ -220,8 +243,6 @@ node[:tile][:data].each_value do |data| group "tile" end elsif file =~ /\.zip$/ - package "unzip" - execute file do action :nothing command "unzip -qq -o #{file} -d #{directory}" @@ -250,7 +271,7 @@ node[:tile][:data].each_value do |data| source url owner "tile" group "tile" - mode 0o644 + mode "644" backup false notifies :run, "execute[#{file}]", :immediately notifies :restart, "service[renderd]" @@ -258,25 +279,26 @@ node[:tile][:data].each_value do |data| end nodejs_package "carto" -nodejs_package "millstone" systemd_service "update-lowzoom@" do description "Low zoom tile update service for %i layer" user "tile" + exec_start_pre "+/bin/systemctl stop render-lowzoom.service" exec_start "/bin/bash /usr/local/bin/update-lowzoom-%i" - private_tmp true - private_devices true - private_network true - protect_system "full" - protect_home true - no_new_privileges true + runtime_directory "update-lowzoom-%i" + sandbox true + restrict_address_families "AF_UNIX" + read_write_paths [ + "/srv/tile.openstreetmap.org/tiles/%i", + "/var/log/tile" + ] restart "on-failure" end directory "/srv/tile.openstreetmap.org/styles" do owner "tile" group "tile" - mode 0o755 + mode "755" end node[:tile][:styles].each do |name, details| @@ -287,7 +309,7 @@ node[:tile][:styles].each do |name, details| source "update-lowzoom.erb" owner "root" group "root" - mode 0o755 + mode "755" variables :style => name end @@ -299,21 +321,21 @@ node[:tile][:styles].each do |name, details| directory tile_directory do owner "tile" group "tile" - mode 0o755 + mode "755" end details[:tile_directories].each do |directory| directory directory[:name] do - owner "www-data" - group "www-data" - mode 0o755 + owner "_renderd" + group "_renderd" + mode "755" end directory[:min_zoom].upto(directory[:max_zoom]) do |zoom| directory "#{directory[:name]}/#{zoom}" do - owner "www-data" - group "www-data" - mode 0o755 + owner "_renderd" + group "_renderd" + mode "755" end link "#{tile_directory}/#{zoom}" do @@ -328,7 +350,7 @@ node[:tile][:styles].each do |name, details| action :create_if_missing owner "tile" group "tile" - mode 0o444 + mode "444" end git style_directory do @@ -345,9 +367,20 @@ node[:tile][:styles].each do |name, details| group "tile" end + if details[:fonts_script] + execute details[:fonts_script] do + action :nothing + command details[:fonts_script] + cwd style_directory + user "tile" + group "tile" + subscribes :run, "git[#{style_directory}]" + end + end + execute "#{style_directory}/project.mml" do action :nothing - command "carto -a 3.0.0 project.mml > project.xml" + command "carto -a 3.0.22 project.mml > project.xml" cwd style_directory user "tile" group "tile" @@ -358,9 +391,9 @@ node[:tile][:styles].each do |name, details| end postgresql_version = node[:tile][:database][:cluster].split("/").first +postgis_version = node[:tile][:database][:postgis] -package "postgis" -package "postgresql-#{postgresql_version}-postgis-2.3" +package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" postgresql_user "jburgess" do cluster node[:tile][:database][:cluster] @@ -372,6 +405,11 @@ postgresql_user "tomh" do superuser true end +postgresql_user "pnorman" do + cluster node[:tile][:database][:cluster] + superuser true +end + postgresql_user "tile" do cluster node[:tile][:database][:cluster] end @@ -380,6 +418,10 @@ postgresql_user "www-data" do cluster node[:tile][:database][:cluster] end +postgresql_user "_renderd" do + cluster node[:tile][:database][:cluster] +end + postgresql_database "gis" do cluster node[:tile][:database][:cluster] owner "tile" @@ -393,9 +435,10 @@ end postgresql_extension "hstore" do cluster node[:tile][:database][:cluster] database "gis" + only_if { node[:tile][:database][:hstore] } end -%w[geography_columns planet_osm_nodes planet_osm_rels planet_osm_ways raster_columns raster_overviews spatial_ref_sys].each do |table| +%w[geography_columns planet_osm_nodes planet_osm_rels planet_osm_ways raster_columns raster_overviews].each do |table| postgresql_table table do cluster node[:tile][:database][:cluster] database "gis" @@ -404,12 +447,37 @@ end end end -%w[geometry_columns planet_osm_line planet_osm_point planet_osm_polygon planet_osm_roads].each do |table| +%w[geometry_columns planet_osm_line planet_osm_point planet_osm_polygon planet_osm_roads spatial_ref_sys].each do |table| postgresql_table table do cluster node[:tile][:database][:cluster] database "gis" owner "tile" - permissions "tile" => :all, "www-data" => :select + permissions "tile" => :all, "www-data" => :select, "_renderd" => :select + end +end + +package %w[ + gdal-bin + python3-yaml + python3-psycopg2 +] + +if node[:tile][:database][:external_data_script] + execute node[:tile][:database][:external_data_script] do + command "#{node[:tile][:database][:external_data_script]} -R _renderd" + cwd "/srv/tile.openstreetmap.org" + user "tile" + group "tile" + ignore_failure true + end + + Array(node[:tile][:database][:external_data_tables]).each do |table| + postgresql_table table do + cluster node[:tile][:database][:cluster] + database "gis" + owner "tile" + permissions "tile" => :all, "www-data" => :select, "_renderd" => :select + end end end @@ -418,90 +486,82 @@ postgresql_munin "gis" do database "gis" end -file node[:tile][:node_file] do +directory File.dirname(node[:tile][:database][:node_file]) do + owner "root" + group "root" + mode "755" + recursive true +end + +file node[:tile][:database][:node_file] do owner "tile" - group "www-data" - mode 0o640 + group "_renderd" + mode "660" end directory "/var/log/tile" do owner "tile" group "tile" - mode 0o755 + mode "755" end package %w[ osm2pgsql - osmosis + osmium-tool + pyosmium + python3-pyproj ] -package %w[ - ruby - ruby-dev -] - -package %w[ - libproj-dev - libxml2-dev -] - -gem_package "proj4rb" -gem_package "libxml-ruby" - -remote_directory "/usr/local/lib/site_ruby" do - source "ruby" +remote_directory "/usr/local/bin" do + source "bin" owner "root" group "root" - mode 0o755 + mode "755" files_owner "root" files_group "root" - files_mode 0o644 + files_mode "755" end template "/usr/local/bin/expire-tiles" do source "expire-tiles.erb" owner "root" group "root" - mode 0o755 + mode "755" end directory "/var/lib/replicate" do owner "tile" group "tile" - mode 0o755 + mode "755" end directory "/var/lib/replicate/expire-queue" do owner "tile" - group "www-data" - mode 0o775 -end - -template "/var/lib/replicate/configuration.txt" do - source "replicate.configuration.erb" - owner "tile" - group "tile" - mode 0o644 + group "_renderd" + mode "775" end template "/usr/local/bin/replicate" do source "replicate.erb" owner "root" group "root" - mode 0o755 + mode "755" + variables :postgresql_version => postgresql_version.to_f end systemd_service "expire-tiles" do description "Tile dirtying service" type "simple" - user "www-data" + user "_renderd" exec_start "/usr/local/bin/expire-tiles" + nice 10 standard_output "null" - private_tmp true - private_devices true - protect_system "full" - protect_home true - no_new_privileges true + sandbox true + read_write_paths tile_directories + [ + "/store/database/nodes", + "/var/lib/replicate/expire-queue", + "/var/log/tile" + ] end systemd_path "expire-tiles" do @@ -520,11 +580,13 @@ systemd_service "replicate" do wants "postgresql.service" user "tile" exec_start "/usr/local/bin/replicate" - private_tmp true - private_devices true - protect_system "full" - protect_home true - no_new_privileges true + sandbox :enable_network => true + restrict_address_families "AF_UNIX" + read_write_paths [ + "/store/database/nodes", + "/var/lib/replicate", + "/var/log/tile" + ] restart "on-failure" end @@ -534,25 +596,30 @@ service "replicate" do subscribes :restart, "systemd_service[replicate]" end -template "/etc/logrotate.d/replicate" do - source "replicate.logrotate.erb" - owner "root" - group "root" - mode 0o644 -end - template "/usr/local/bin/render-lowzoom" do source "render-lowzoom.erb" owner "root" group "root" - mode 0o755 + mode "755" end -template "/etc/cron.d/render-lowzoom" do - source "render-lowzoom.cron.erb" - owner "root" - group "root" - mode 0o644 +systemd_service "render-lowzoom" do + description "Render low zoom tiles" + condition_path_exists_glob "!/run/update-lowzoom-*" + user "tile" + exec_start "/usr/local/bin/render-lowzoom" + sandbox true + restrict_address_families "AF_UNIX" + read_write_paths "/var/log/tile" +end + +systemd_timer "render-lowzoom" do + description "Render low zoom tiles" + on_calendar "Fri *-*-* 23:00:00 UTC" +end + +service "render-lowzoom.timer" do + action [:enable, :start] end package "liblockfile-simple-perl" @@ -562,19 +629,31 @@ template "/usr/local/bin/cleanup-tiles" do source "cleanup-tiles.erb" owner "root" group "root" - mode 0o755 + mode "755" end -tile_directories = node[:tile][:styles].collect do |_, style| - style[:tile_directories].collect { |directory| directory[:name] } -end.flatten.sort.uniq +systemd_service "cleanup-tiles@" do + description "Cleanup old tiles for /%I" + exec_start "/usr/local/bin/cleanup-tiles /%I" + user "_renderd" + io_scheduling_class "idle" + sandbox true + read_write_paths "/%I" +end -template "/etc/cron.d/cleanup-tiles" do - source "cleanup-tiles.cron.erb" - owner "root" - group "root" - mode 0o644 - variables :directories => tile_directories +systemd_timer "cleanup-tiles@" do + description "Cleanup old tiles for /%I" + on_boot_sec "30m" + on_unit_inactive_sec "60m" + randomized_delay_sec "10m" +end + +tile_directories.each do |directory| + label = directory[1..].gsub("/", "-") + + service "cleanup-tiles@#{label}.timer" do + action [:enable, :start] + end end munin_plugin "mod_tile_fresh" @@ -588,6 +667,14 @@ munin_plugin "renderd_queue_time" munin_plugin "renderd_zoom" munin_plugin "renderd_zoom_time" -munin_plugin "replication_delay" do - conf "munin.erb" +munin_plugin "replication_delay" + +package "ruby-webrick" + +prometheus_exporter "modtile" do + port 9494 +end + +prometheus_exporter "renderd" do + port 9393 end