X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/a62943629fc45519becfbd66176c9238acbeacbd..6a1f3b40b51cfad8ef65a8c3a45815a854b6a6bc:/cookbooks/nominatim/recipes/default.rb?ds=sidebyside diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 285d795f1..75f06fa56 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -20,6 +20,7 @@ include_recipe "accounts" include_recipe "munin" include_recipe "php::fpm" +include_recipe "prometheus" basedir = data_bag_item("accounts", "nominatim")["home"] email_errors = data_bag_item("accounts", "lonvia")["email"] @@ -31,6 +32,8 @@ directory basedir do recursive true end +## Log directory setup + directory node[:nominatim][:logdir] do owner "nominatim" group "nominatim" @@ -52,17 +55,6 @@ file "#{node[:nominatim][:logdir]}/update.log" do mode "664" end -# exception granted for a limited time so that they can set up their own server -firewall_rule "increase-limits-gnome-proxy" do - action :accept - family "inet" - source "net:8.43.85.23" - dest "fw" - proto "tcp:syn" - dest_ports "https" - rate_limit "s:10/sec:30" -end - ## Postgresql include_recipe "postgresql" @@ -123,30 +115,6 @@ node[:nominatim][:tablespaces].each do |name, location| end end -if node[:nominatim][:state] == "master" - postgresql_user "replication" do - cluster node[:nominatim][:dbcluster] - password data_bag_item("nominatim", "passwords")["replication"] - replication true - end - - directory node[:rsyncd][:modules][:archive][:path] do - owner "postgres" - group "postgres" - mode "700" - end - - template "/usr/local/bin/clean-db-nominatim" do - source "clean-db-nominatim.erb" - owner "root" - group "root" - mode "755" - variables :archive_dir => node[:rsyncd][:modules][:archive][:path], - :update_stop_file => "#{basedir}/status/updates_disabled", - :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") - end -end - ## Nominatim backend include_recipe "git" @@ -160,27 +128,55 @@ package %w[ libboost-filesystem-dev libexpat1-dev zlib1g-dev - libxml2-dev libbz2-dev libpq-dev - libgeos++-dev libproj-dev + liblua5.3-dev + lua5.3 python3-pyosmium - pyosmium python3-psycopg2 + python3-dotenv + python3-psutil + python3-jinja2 + python3-icu + python3-datrie + python3-yaml php-pgsql php-intl + ruby + ruby-file-tail + ruby-pg + ruby-webrick ] -source_directory = "#{basedir}/nominatim" -build_directory = "#{basedir}/bin" +source_directory = "#{basedir}/src/nominatim" +build_directory = "#{basedir}/src/build" +project_directory = "#{basedir}/planet-project" +bin_directory = "#{basedir}/bin" +cfg_directory = "#{basedir}/etc" ui_directory = "#{basedir}/ui" +qa_bin_directory = "#{basedir}/src/Nominatim-Data-Analyser" +qa_data_directory = "#{basedir}/qa-data" + +[basedir, "#{basedir}/src", cfg_directory, bin_directory, build_directory, project_directory].each do |path| + directory path do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end +end -directory build_directory do +directory "#{bin_directory}/maintenance" do owner "nominatim" group "nominatim" - mode "755" - recursive true + mode "775" +end + +if node[:nominatim][:flatnode_file] + directory File.dirname(node[:nominatim][:flatnode_file]) do + recursive true + end end # Normally syncing via chef is a bad idea because syncing might involve @@ -196,7 +192,15 @@ git source_directory do user "nominatim" group "nominatim" not_if { node[:nominatim][:state] != "slave" && File.exist?("#{source_directory}/README.md") } - notifies :run, "execute[compile_nominatim]", :immediately + notifies :run, "execute[compile_nominatim]" +end + +remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do + action :create_if_missing + source "https://www.nominatim.org/data/country_grid.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" end execute "compile_nominatim" do @@ -204,65 +208,33 @@ execute "compile_nominatim" do user "nominatim" cwd build_directory command "cmake #{source_directory} && make" + notifies :run, "execute[install_nominatim]" end -template "#{source_directory}/.git/hooks/post-merge" do - source "git-post-merge-hook.erb" - owner "nominatim" - group "nominatim" - mode "755" - variables :srcdir => source_directory, - :builddir => build_directory, - :dbname => node[:nominatim][:dbname] +execute "install_nominatim" do + action :nothing + cwd build_directory + command "make install" end -template "#{build_directory}/settings/local.php" do - source "settings.erb" +# Project directory + +template "#{project_directory}/.env" do + source "nominatim.env.erb" owner "nominatim" group "nominatim" mode "664" variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", :dbname => node[:nominatim][:dbname], :flatnode_file => node[:nominatim][:flatnode_file], - :log_file => "#{node[:nominatim][:logdir]}/query.log" -end - -git ui_directory do - action :sync - repository node[:nominatim][:ui_repository] - revision node[:nominatim][:ui_revision] - user "nominatim" - group "nominatim" -end - -template "#{ui_directory}/dist/config.js" do - source "ui-config.js.erb" - owner "nominatim" - group "nominatim" - mode "664" -end - -if node[:nominatim][:flatnode_file] - directory File.dirname(node[:nominatim][:flatnode_file]) do - recursive true - end + :log_file => "#{node[:nominatim][:logdir]}/query.log", + :tokenizer => node[:nominatim][:config][:tokenizer], + :forward_dependencies => node[:nominatim][:config][:forward_dependencies] end -template "/etc/logrotate.d/nominatim" do - source "logrotate.nominatim.erb" - owner "root" - group "root" - mode "644" -end - -external_data = [ - "wikimedia-importance.sql.gz", - "gb_postcode_data.sql.gz" -] - -external_data.each do |fname| - remote_file "#{source_directory}/data/#{fname}" do - action :create_if_missing +%w[wikimedia-importance.sql.gz gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| + remote_file "#{project_directory}/#{fname}" do + action :create source "https://www.nominatim.org/data/#{fname}" owner "nominatim" group "nominatim" @@ -270,108 +242,10 @@ external_data.each do |fname| end end -remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do - action :create_if_missing - source "https://www.nominatim.org/data/country_grid.sql.gz" - owner "nominatim" - group "nominatim" - mode "644" -end - -if node[:nominatim][:state] == "off" - cron_d "nominatim-backup" do - action :delete - end - - cron_d "nominatim-vacuum-db" do - action :delete - end - - cron_d "nominatim-clean-db" do - action :delete - end - - cron_d "nominatim-update-maintenance-trigger" do - action :delete - end -else - cron_d "nominatim-backup" do - action node[:nominatim][:enable_backup] ? :create : :delete - minute "0" - hour "3" - day "1" - user "nominatim" - command "/usr/local/bin/backup-nominatim" - mailto email_errors - end - - cron_d "nominatim-vacuum-db" do - minute "20" - hour "0" - user "postgres" - command "/usr/local/bin/vacuum-db-nominatim" - mailto email_errors - end - - cron_d "nominatim-clean-db" do - action node[:nominatim][:state] == "master" ? :create : :delete - minute "5" - hour "*/4" - user "postgres" - command "/usr/local/bin/clean-db-nominatim" - mailto email_errors - end - - cron_d "nominatim-update-maintenance-trigger" do - minute "18" - hour "1" - user "nominatim" - command "touch #{basedir}/status/update_maintenance" - mailto email_errors - end -end - -template "#{source_directory}/utils/nominatim-update" do - source "updater.erb" - user "nominatim" - group "nominatim" - mode "755" - variables :bindir => build_directory, - :srcdir => source_directory, - :logfile => "#{node[:nominatim][:logdir]}/update.log", - :branch => node[:nominatim][:revision], - :update_stop_file => "#{basedir}/status/updates_disabled", - :update_maintenance_trigger => "#{basedir}/status/update_maintenance" -end - -template "/etc/init.d/nominatim-update" do - source "updater.init.erb" - user "nominatim" - group "nominatim" - mode "755" - variables :source_directory => source_directory -end - -%w[backup-nominatim vacuum-db-nominatim].each do |fname| - template "/usr/local/bin/#{fname}" do - source "#{fname}.erb" - owner "root" - group "root" - mode "755" - variables :db => node[:nominatim][:dbname] - end -end - -## webserver frontend - -directory "#{basedir}/etc" do - owner "nominatim" - group "adm" - mode "775" -end +# Webserver + frontend %w[user_agent referrer email generic].each do |name| - file "#{basedir}/etc/nginx_blocked_#{name}.conf" do + file "#{cfg_directory}/nginx_blocked_#{name}.conf" do action :create_if_missing owner "nominatim" group "adm" @@ -380,7 +254,7 @@ end end node[:nominatim][:fpm_pools].each do |name, data| - php_fpm name.to_s do + php_fpm name do port data[:port] pm data[:pm] pm_max_children data[:max_children] @@ -388,6 +262,7 @@ node[:nominatim][:fpm_pools].each do |name, data| pm_min_spare_servers 10 pm_max_spare_servers 20 pm_max_requests 10000 + prometheus_port data[:prometheus_port] end end @@ -398,14 +273,11 @@ ssl_certificate node[:fqdn] do "nominatim.openstreetmap.com", "nominatim.openstreetmap.net", "nominatim.openstreetmaps.org", - "nominatim.openmaps.org"] + "nominatim.openmaps.org", + "nominatim.qgis.org"] notifies :reload, "service[nginx]" end -package "apache2" do - action :remove -end - include_recipe "nginx" nginx_site "default" do @@ -416,7 +288,7 @@ frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) nginx_site "nominatim" do template "nginx.erb" - directory build_directory + directory project_directory variables :pools => node[:nominatim][:fpm_pools], :frontends => frontends, :confdir => "#{basedir}/etc", @@ -430,6 +302,195 @@ template "/etc/logrotate.d/nginx" do mode "644" end +# Updates + +%w[nominatim-update + nominatim-update-source + nominatim-update-refresh-db + nominatim-update-data + nominatim-daily-maintenance].each do |fname| + template "#{bin_directory}/#{fname}" do + source "#{fname}.erb" + owner "nominatim" + group "nominatim" + mode "554" + variables :bindir => bin_directory, + :srcdir => source_directory, + :builddir => build_directory, + :projectdir => project_directory, + :qabindir => qa_bin_directory, + :qadatadir => qa_data_directory + end +end + +systemd_service "nominatim-update" do + description "Update the Nominatim database" + exec_start "#{bin_directory}/nominatim-update" + restart "on-success" + standard_output "append:#{node[:nominatim][:logdir]}/update.log" + standard_error "inherit" + working_directory project_directory +end + +systemd_service "nominatim-update-maintenance-trigger" do + description "Trigger daily maintenance tasks for Nominatim DB" + exec_start "ln -sf #{bin_directory}/nominatim-daily-maintenance #{bin_directory}/maintenance/" + user "nominatim" +end + +systemd_timer "nominatim-update-maintenance-trigger" do + action node[:nominatim][:state] != "off" ? :create : :delete + description "Schedule daily maintenance tasks for Nominatim DB" + on_calendar "*-*-* 02:03:00 UTC" +end + +service "nominatim-update-maintenance-trigger" do + action node[:nominatim][:state] != "off" ? :enable : :disable +end + +# Nominatim UI + +git ui_directory do + action :sync + repository node[:nominatim][:ui_repository] + revision node[:nominatim][:ui_revision] + user "nominatim" + group "nominatim" +end + +template "#{ui_directory}/dist/theme/config.theme.js" do + source "ui-config.js.erb" + owner "nominatim" + group "nominatim" + mode "664" +end + +# Nominatim QA + +if node[:nominatim][:enable_qa_tiles] + package "python3-geojson" + + git qa_bin_directory do + repository node[:nominatim][:qa_repository] + revision node[:nominatim][:qa_revision] + enable_submodules true + user "nominatim" + group "nominatim" + notifies :run, "execute[compile_qa]" + end + + execute "compile_qa" do + action :nothing + user "nominatim" + cwd "#{qa_bin_directory}/clustering-vt" + command "make" + end + + directory qa_data_directory do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end + + template "#{qa_bin_directory}/analyser/config/config.yaml" do + source "qa_config.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :outputdir => "#{qa_data_directory}/new" + end + + ssl_certificate "qa-tile.nominatim.openstreetmap.org" do + domains ["qa-tile.nominatim.openstreetmap.org"] + notifies :reload, "service[nginx]" + end + + nginx_site "qa-tiles.nominatim" do + template "nginx-qa-tiles.erb" + directory build_directory + variables :qa_data_directory => qa_data_directory + end + +end + +# Replication + +cron_d "nominatim-clean-db" do + action node[:nominatim][:state] == "master" ? :create : :delete + minute "5" + hour "*/4" + user "postgres" + command "#{bin_directory}/clean-db-nominatim" + mailto email_errors +end + +if node[:nominatim][:state] == "master" + postgresql_user "replication" do + cluster node[:nominatim][:dbcluster] + password data_bag_item("nominatim", "passwords")["replication"] + replication true + end + + directory node[:rsyncd][:modules][:archive][:path] do + owner "postgres" + group "postgres" + mode "700" + end + + template "#{bin_directory}/clean-db-nominatim" do + source "clean-db-nominatim.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :archive_dir => node[:rsyncd][:modules][:archive][:path], + :update_stop_file => "#{basedir}/status/updates_disabled", + :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") + end +end + +# Maintenance + +cron_d "nominatim-backup" do + action (node[:nominatim][:enable_backup] && node[:nominatim][:state] != "off") ? :create : :delete + minute "0" + hour "3" + day "1" + user "nominatim" + command "#{bin_directory}/backup-nominatim" + mailto email_errors +end + +cron_d "nominatim-vacuum-db" do + action node[:nominatim][:state] != "off" ? :create : :delete + minute "20" + hour "0" + user "postgres" + command "#{bin_directory}/vacuum-db-nominatim" + mailto email_errors +end + +%w[backup-nominatim vacuum-db-nominatim].each do |fname| + template "#{bin_directory}/#{fname}" do + source "#{fname}.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :db => node[:nominatim][:dbname] + end +end + +# Logging + +template "/etc/logrotate.d/nominatim" do + source "logrotate.nominatim.erb" + owner "root" + group "root" + mode "644" +end + +# Monitoring + munin_plugin_conf "nominatim" do template "munin.erb" variables :db => node[:nominatim][:dbname], @@ -448,10 +509,13 @@ munin_plugin "nominatim_requests" do target "#{source_directory}/munin/nominatim_requests_querylog" end -directory "#{basedir}/status" do - owner "nominatim" - group "postgres" - mode "775" +prometheus_exporter "nominatim" do + port 8082 + user "www-data" + options [ + "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log", + "--nominatim.database-name=#{node[:nominatim][:dbname]}" + ] end include_recipe "fail2ban" @@ -462,6 +526,6 @@ fail2ban_jail "nominatim_limit_req" do filter "nginx-limit-req" logpath "#{node[:nominatim][:logdir]}/nominatim.openstreetmap.org-error.log" ports [80, 443] - maxretry 5 + maxretry 20 ignoreips frontend_addresses.flatten.sort end