X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/1e07aa23bfd2bcc46dd6a82855d9f316a17a39ca..f65e6e5bdfdab789a385847963b9c1ffa799a6e3:/cookbooks/nominatim/recipes/default.rb diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 0e602943f..b5496a652 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -1,14 +1,14 @@ # -# Cookbook Name:: nominatim +# Cookbook:: nominatim # Recipe:: base # -# Copyright 2015, OpenStreetMap Foundation +# Copyright:: 2015, OpenStreetMap Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -17,20 +17,24 @@ # limitations under the License. # +include_recipe "accounts" +include_recipe "munin" +include_recipe "php::fpm" + basedir = data_bag_item("accounts", "nominatim")["home"] email_errors = data_bag_item("accounts", "lonvia")["email"] directory basedir do owner "nominatim" group "nominatim" - mode 0o755 + mode "755" recursive true end directory node[:nominatim][:logdir] do owner "nominatim" group "nominatim" - mode 0o755 + mode "755" recursive true end @@ -38,21 +42,35 @@ file "#{node[:nominatim][:logdir]}/query.log" do action :create_if_missing owner "www-data" group "adm" - mode 0o664 + mode "664" end file "#{node[:nominatim][:logdir]}/update.log" do action :create_if_missing owner "nominatim" group "adm" - mode 0o664 + mode "664" +end + +# exception granted for a limited time so that they can set up their own server +firewall_rule "increase-limits-gnome-proxy" do + action :accept + family "inet" + source "net:8.43.85.23" + dest "fw" + proto "tcp:syn" + dest_ports "https" + rate_limit "s:10/sec:30" end ## Postgresql include_recipe "postgresql" -package "postgis" +postgresql_version = node[:nominatim][:dbcluster].split("/").first +postgis_version = node[:nominatim][:postgis] + +package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" node[:nominatim][:dbadmins].each do |user| postgresql_user user do @@ -81,17 +99,17 @@ end directory "#{basedir}/tablespaces" do owner "postgres" group "postgres" - mode 0o700 + mode "700" end -# Note: tablespaces must be exactly in the same location on each +# NOTE: tablespaces must be exactly in the same location on each # Nominatim instance when replication is in use. Therefore # use symlinks to canonical directory locations. node[:nominatim][:tablespaces].each do |name, location| directory location do owner "postgres" group "postgres" - mode 0o700 + mode "700" recursive true end @@ -105,7 +123,7 @@ node[:nominatim][:tablespaces].each do |name, location| end end -if node[:nominatim][:state] == "master" # ~FC023 +if node[:nominatim][:state] == "master" postgresql_user "replication" do cluster node[:nominatim][:dbcluster] password data_bag_item("nominatim", "passwords")["replication"] @@ -115,14 +133,14 @@ if node[:nominatim][:state] == "master" # ~FC023 directory node[:rsyncd][:modules][:archive][:path] do owner "postgres" group "postgres" - mode 0o700 + mode "700" end template "/usr/local/bin/clean-db-nominatim" do source "clean-db-nominatim.erb" owner "root" group "root" - mode 0o755 + mode "755" variables :archive_dir => node[:rsyncd][:modules][:archive][:path], :update_stop_file => "#{basedir}/status/updates_disabled", :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") @@ -132,33 +150,36 @@ end ## Nominatim backend include_recipe "git" -include_recipe "python" - -package "build-essential" -package "cmake" -package "g++" -package "libboost-dev" -package "libboost-system-dev" -package "libboost-filesystem-dev" -package "libboost-python-dev" -package "libexpat1-dev" -package "zlib1g-dev" -package "libxml2-dev" -package "libbz2-dev" -package "libpq-dev" -package "libgeos++-dev" -package "libproj-dev" -package "osmosis" - -python_package "osmium" + +package %w[ + build-essential + cmake + g++ + libboost-dev + libboost-system-dev + libboost-filesystem-dev + libexpat1-dev + zlib1g-dev + libxml2-dev + libbz2-dev + libpq-dev + libgeos++-dev + libproj-dev + python3-pyosmium + pyosmium + python3-psycopg2 + php-pgsql + php-intl +] source_directory = "#{basedir}/nominatim" build_directory = "#{basedir}/bin" +ui_directory = "#{basedir}/ui" directory build_directory do owner "nominatim" group "nominatim" - mode 0o755 + mode "755" recursive true end @@ -174,6 +195,7 @@ git source_directory do enable_submodules true user "nominatim" group "nominatim" + not_if { node[:nominatim][:state] != "slave" && File.exist?("#{source_directory}/README.md") } notifies :run, "execute[compile_nominatim]", :immediately end @@ -188,7 +210,7 @@ template "#{source_directory}/.git/hooks/post-merge" do source "git-post-merge-hook.erb" owner "nominatim" group "nominatim" - mode 0o755 + mode "755" variables :srcdir => source_directory, :builddir => build_directory, :dbname => node[:nominatim][:dbname] @@ -198,14 +220,29 @@ template "#{build_directory}/settings/local.php" do source "settings.erb" owner "nominatim" group "nominatim" - mode 0o664 + mode "664" variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", :dbname => node[:nominatim][:dbname], :flatnode_file => node[:nominatim][:flatnode_file], :log_file => "#{node[:nominatim][:logdir]}/query.log" end -if node[:nominatim][:flatnode_file] # ~FC023 +git ui_directory do + action :sync + repository node[:nominatim][:ui_repository] + revision node[:nominatim][:ui_revision] + user "nominatim" + group "nominatim" +end + +template "#{ui_directory}/dist/config.js" do + source "ui-config.js.erb" + owner "nominatim" + group "nominatim" + mode "664" +end + +if node[:nominatim][:flatnode_file] directory File.dirname(node[:nominatim][:flatnode_file]) do recursive true end @@ -215,162 +252,183 @@ template "/etc/logrotate.d/nominatim" do source "logrotate.nominatim.erb" owner "root" group "root" - mode 0o644 + mode "644" end external_data = [ - "wikipedia_article.sql.bin", - "wikipedia_redirect.sql.bin", + "wikimedia-importance.sql.gz", "gb_postcode_data.sql.gz" ] external_data.each do |fname| remote_file "#{source_directory}/data/#{fname}" do action :create_if_missing - source "http://www.nominatim.org/data/#{fname}" + source "https://www.nominatim.org/data/#{fname}" owner "nominatim" group "nominatim" - mode 0o644 + mode "644" end end -template "/etc/cron.d/nominatim" do - action node[:nominatim][:state] == :off ? :delete : :create - source "nominatim.cron.erb" - owner "root" - group "root" - mode "0644" - variables :bin_directory => "#{source_directory}/utils", :mailto => email_errors +remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do + action :create_if_missing + source "https://www.nominatim.org/data/country_grid.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" +end + +if node[:nominatim][:state] == "off" + cron_d "nominatim-backup" do + action :delete + end + + cron_d "nominatim-vacuum-db" do + action :delete + end + + cron_d "nominatim-clean-db" do + action :delete + end + + cron_d "nominatim-update-maintenance-trigger" do + action :delete + end +else + cron_d "nominatim-backup" do + action node[:nominatim][:enable_backup] ? :create : :delete + minute "0" + hour "3" + day "1" + user "nominatim" + command "/usr/local/bin/backup-nominatim" + mailto email_errors + end + + cron_d "nominatim-vacuum-db" do + minute "20" + hour "0" + user "postgres" + command "/usr/local/bin/vacuum-db-nominatim" + mailto email_errors + end + + cron_d "nominatim-clean-db" do + action node[:nominatim][:state] == "master" ? :create : :delete + minute "5" + hour "*/4" + user "postgres" + command "/usr/local/bin/clean-db-nominatim" + mailto email_errors + end + + cron_d "nominatim-update-maintenance-trigger" do + minute "18" + hour "1" + user "nominatim" + command "touch #{basedir}/status/update_maintenance" + mailto email_errors + end end template "#{source_directory}/utils/nominatim-update" do source "updater.erb" user "nominatim" group "nominatim" - mode 0o755 + mode "755" variables :bindir => build_directory, :srcdir => source_directory, :logfile => "#{node[:nominatim][:logdir]}/update.log", :branch => node[:nominatim][:revision], - :update_stop_file => "#{basedir}/status/updates_disabled" + :update_stop_file => "#{basedir}/status/updates_disabled", + :update_maintenance_trigger => "#{basedir}/status/update_maintenance" end template "/etc/init.d/nominatim-update" do source "updater.init.erb" user "nominatim" group "nominatim" - mode 0o755 + mode "755" variables :source_directory => source_directory end -%w(backup-nominatim vacuum-db-nominatim).each do |fname| +%w[backup-nominatim vacuum-db-nominatim].each do |fname| template "/usr/local/bin/#{fname}" do source "#{fname}.erb" owner "root" group "root" - mode 0o755 + mode "755" variables :db => node[:nominatim][:dbname] end end ## webserver frontend -template "#{build_directory}/settings/ip_blocks.conf" do - action :create_if_missing - source "ipblocks.erb" - owner "nominatim" - group "nominatim" - mode 0o664 -end - -file "#{build_directory}/settings/apache_blocks.conf" do - action :create_if_missing - owner "nominatim" - group "nominatim" - mode 0o664 -end - -file "#{build_directory}/settings/ip_blocks.map" do - action :create_if_missing +directory "#{basedir}/etc" do owner "nominatim" - group "nominatim" - mode 0o664 + group "adm" + mode "775" end -include_recipe "apache" - -package "php" -package "php-fpm" -package "php-pgsql" -package "php-pear" -package "php-db" - -apache_module "rewrite" -apache_module "proxy" -apache_module "proxy_fcgi" -apache_module "proxy_http" -apache_module "headers" -apache_module "reqtimeout" - -service "php7.0-fpm" do - action [:enable, :start] - supports :status => true, :restart => true, :reload => true +%w[user_agent referrer email generic].each do |name| + file "#{basedir}/etc/nginx_blocked_#{name}.conf" do + action :create_if_missing + owner "nominatim" + group "adm" + mode "664" + end end node[:nominatim][:fpm_pools].each do |name, data| - template "/etc/php/7.0/fpm/pool.d/#{name}.conf" do - source "fpm.conf.erb" - owner "root" - group "root" - mode 0o644 - variables data.merge(:name => name) - notifies :reload, "service[php7.0-fpm]" + php_fpm name do + port data[:port] + pm data[:pm] + pm_max_children data[:max_children] + pm_start_servers 20 + pm_min_spare_servers 10 + pm_max_spare_servers 20 + pm_max_requests 10000 + prometheus_port data[:prometheus_port] end end -ssl_certificate "nominatim.openstreetmap.org" do - domains ["nominatim.openstreetmap.org", +ssl_certificate node[:fqdn] do + domains [node[:fqdn], + "nominatim.openstreetmap.org", "nominatim.osm.org", "nominatim.openstreetmap.com", "nominatim.openstreetmap.net", "nominatim.openstreetmaps.org", "nominatim.openmaps.org"] - notifies :reload, "service[apache2]" + notifies :reload, "service[nginx]" end -apache_site "nominatim.openstreetmap.org" do - template "apache.erb" - directory build_directory - variables :pools => node[:nominatim][:fpm_pools] +package "apache2" do + action :remove end -apache_site "default" do - action [:disable] -end +include_recipe "nginx" -template "/etc/logrotate.d/apache2" do - source "logrotate.apache.erb" - owner "root" - group "root" - mode 0o644 +nginx_site "default" do + action [:delete] end -include_recipe "fail2ban" +frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) -web_servers = search(:node, "recipes:web\\:\\:frontend").collect do |n| # ~FC010 - n.ipaddresses(:role => :external) -end.flatten - -fail2ban_filter "nominatim" do - failregex '^ - - \[\] "[^"]+" (408|429) ' +nginx_site "nominatim" do + template "nginx.erb" + directory build_directory + variables :pools => node[:nominatim][:fpm_pools], + :frontends => frontends, + :confdir => "#{basedir}/etc", + :ui_directory => ui_directory end -fail2ban_jail "nominatim" do - filter "nominatim" - logpath "/var/log/apache2/nominatim.openstreetmap.org-access.log" - ports [80, 443] - maxretry 100 - ignoreips web_servers +template "/etc/logrotate.d/nginx" do + source "logrotate.nginx.erb" + owner "root" + group "root" + mode "644" end munin_plugin_conf "nominatim" do @@ -391,12 +449,20 @@ munin_plugin "nominatim_requests" do target "#{source_directory}/munin/nominatim_requests_querylog" end -munin_plugin "nominatim_throttled_ips" do - target "#{source_directory}/munin/nominatim_throttled_ips" -end - directory "#{basedir}/status" do owner "nominatim" group "postgres" - mode 0o775 + mode "775" +end + +include_recipe "fail2ban" + +frontend_addresses = frontends.collect { |f| f.ipaddresses(:role => :external) } + +fail2ban_jail "nominatim_limit_req" do + filter "nginx-limit-req" + logpath "#{node[:nominatim][:logdir]}/nominatim.openstreetmap.org-error.log" + ports [80, 443] + maxretry 5 + ignoreips frontend_addresses.flatten.sort end