X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/68e068818d559ef35bdf2a138a93596337828ef9..794cc957324de66edab45373053be3e601090f1f:/cookbooks/nominatim/recipes/default.rb diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 537de83f5..7eb6a1e74 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -18,12 +18,19 @@ # include_recipe "accounts" -include_recipe "munin" -include_recipe "php::fpm" include_recipe "prometheus" +include_recipe "postgresql" +include_recipe "python" +include_recipe "nginx" +include_recipe "git" +include_recipe "fail2ban" basedir = data_bag_item("accounts", "nominatim")["home"] -email_errors = data_bag_item("accounts", "lonvia")["email"] +project_directory = "#{basedir}/planet-project" +bin_directory = "#{basedir}/bin" +cfg_directory = "#{basedir}/etc" +ui_directory = "#{basedir}/ui" +qa_data_directory = "#{basedir}/qa-data" directory basedir do owner "nominatim" @@ -32,6 +39,26 @@ directory basedir do recursive true end +[basedir, bin_directory, cfg_directory, project_directory, ui_directory].each do |path| + directory path do + owner "nominatim" + group "nominatim" + mode "755" + end +end + +if node[:nominatim][:flatnode_file] + directory File.dirname(node[:nominatim][:flatnode_file]) do + recursive true + end +end + +directory "#{bin_directory}/maintenance" do + owner "nominatim" + group "nominatim" + mode "775" +end + ## Log directory setup directory node[:nominatim][:logdir] do @@ -48,16 +75,7 @@ file "#{node[:nominatim][:logdir]}/query.log" do mode "664" end -file "#{node[:nominatim][:logdir]}/update.log" do - action :create_if_missing - owner "nominatim" - group "adm" - mode "664" -end - -## Postgresql - -include_recipe "postgresql" +### Postgresql postgresql_version = node[:nominatim][:dbcluster].split("/").first postgis_version = node[:nominatim][:postgis] @@ -68,174 +86,94 @@ node[:nominatim][:dbadmins].each do |user| postgresql_user user do cluster node[:nominatim][:dbcluster] superuser true - only_if { node[:nominatim][:state] != "slave" } end end postgresql_user "nominatim" do cluster node[:nominatim][:dbcluster] superuser true - only_if { node[:nominatim][:state] != "slave" } end postgresql_user "www-data" do cluster node[:nominatim][:dbcluster] - only_if { node[:nominatim][:state] != "slave" } end -postgresql_munin "nominatim" do - cluster node[:nominatim][:dbcluster] - database node[:nominatim][:dbname] -end +### Nominatim -directory "#{basedir}/tablespaces" do - owner "postgres" - group "postgres" - mode "700" -end - -# NOTE: tablespaces must be exactly in the same location on each -# Nominatim instance when replication is in use. Therefore -# use symlinks to canonical directory locations. -node[:nominatim][:tablespaces].each do |name, location| - directory location do - owner "postgres" - group "postgres" - mode "700" - recursive true - end - - link "#{basedir}/tablespaces/#{name}" do - to location - end - - postgresql_tablespace name do - cluster node[:nominatim][:dbcluster] - location "#{basedir}/tablespaces/#{name}" - end -end - -## Nominatim backend - -include_recipe "git" +python_directory = "#{basedir}/venv" package %w[ build-essential - cmake - g++ - libboost-dev - libboost-system-dev - libboost-filesystem-dev - libexpat1-dev - zlib1g-dev - libbz2-dev - libpq-dev - libproj-dev - liblua5.3-dev - lua5.3 - python3-pyosmium - python3-psycopg2 - python3-dotenv - python3-psutil - python3-jinja2 - python3-icu - python3-datrie - python3-yaml - php-pgsql - php-intl + libicu-dev + python3-dev + pkg-config + osm2pgsql ruby ruby-file-tail ruby-pg ruby-webrick ] -source_directory = "#{basedir}/src/nominatim" -build_directory = "#{basedir}/src/build" -project_directory = "#{basedir}/planet-project" -bin_directory = "#{basedir}/bin" -cfg_directory = "#{basedir}/etc" -ui_directory = "#{basedir}/ui" -qa_bin_directory = "#{basedir}/src/Nominatim-Data-Analyser" -qa_data_directory = "#{basedir}/qa-data" - -[basedir, "#{basedir}/src", cfg_directory, bin_directory, build_directory, project_directory].each do |path| - directory path do - owner "nominatim" - group "nominatim" - mode "755" - recursive true - end -end - -directory "#{bin_directory}/maintenance" do - owner "nominatim" - group "nominatim" - mode "775" +python_virtualenv python_directory do + interpreter "/usr/bin/python3" end -if node[:nominatim][:flatnode_file] - directory File.dirname(node[:nominatim][:flatnode_file]) do - recursive true - end +# These are updated during the database update. +python_package "nominatim-db" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] end -# Normally syncing via chef is a bad idea because syncing might involve -# an update of database functions which should not be done while an update -# is ongoing. Therefore we sync in between update cycles. There is an -# exception for slaves: they get DB function updates from the master, so -# only the source code needs to be updated, which chef may do. -git source_directory do - action node[:nominatim][:state] == "slave" ? :sync : :checkout - repository node[:nominatim][:repository] - revision node[:nominatim][:revision] - enable_submodules true - user "nominatim" - group "nominatim" - not_if { node[:nominatim][:state] != "slave" && File.exist?("#{source_directory}/README.md") } - notifies :run, "execute[compile_nominatim]" +python_package "nominatim-api" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] end -remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do - action :create_if_missing - source "https://www.nominatim.org/data/country_grid.sql.gz" +remote_directory "#{project_directory}/static-website" do + source "website" owner "nominatim" group "nominatim" - mode "644" -end - -execute "compile_nominatim" do - action :nothing - user "nominatim" - cwd build_directory - command "cmake #{source_directory} && make" - notifies :run, "execute[install_nominatim]" -end - -execute "install_nominatim" do - action :nothing - cwd build_directory - command "make install" + mode "755" + files_owner "nominatim" + files_group "nominatim" + files_mode "644" + purge false end -# Project directory - template "#{project_directory}/.env" do source "nominatim.env.erb" owner "nominatim" group "nominatim" mode "664" - variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", + variables :base_url => "nominatim.openstreetmap.org", :dbname => node[:nominatim][:dbname], :flatnode_file => node[:nominatim][:flatnode_file], :log_file => "#{node[:nominatim][:logdir]}/query.log", - :tokenizer => node[:nominatim][:config][:tokenizer], - :forward_dependencies => node[:nominatim][:config][:forward_dependencies] + :pool_size => node[:nominatim][:api_pool_size], + :query_timeout => node[:nominatim][:api_query_timeout], + :request_timeout => node[:nominatim][:api_request_timeout] +end + +remote_file "#{project_directory}/secondary_importance.sql.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-secondary-importance.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" +end + +remote_file "#{project_directory}/wikimedia-importance.csv.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-importance.csv.gz" + owner "nominatim" + group "nominatim" + mode "644" end -%w[wikimedia-importance.sql.gz gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| +%w[gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| remote_file "#{project_directory}/#{fname}" do action :create - source "https://www.nominatim.org/data/#{fname}" + source "https://nominatim.org/data/#{fname}" owner "nominatim" group "nominatim" mode "644" @@ -253,17 +191,26 @@ end end end -node[:nominatim][:fpm_pools].each do |name, data| - php_fpm name do - port data[:port] - pm data[:pm] - pm_max_children data[:max_children] - pm_start_servers 20 - pm_min_spare_servers 10 - pm_max_spare_servers 20 - pm_max_requests 10000 - prometheus_port data[:prometheus_port] - end +systemd_service "nominatim" do + description "Nominatim running as a gunicorn application" + user "www-data" + group "www-data" + working_directory project_directory + standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log" + standard_error "inherit" + exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker 'nominatim_api.server.falcon.server:run_wsgi()'" + exec_reload "/bin/kill -s HUP $MAINPID" + kill_mode "mixed" + timeout_stop_sec 5 + private_tmp true + requires "nominatim.socket" + after "network.target" +end + +systemd_socket "nominatim" do + description "Gunicorn socket for Nominatim" + listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock" + socket_user "www-data" end ssl_certificate node[:fqdn] do @@ -278,8 +225,6 @@ ssl_certificate node[:fqdn] do notifies :reload, "service[nginx]" end -include_recipe "nginx" - nginx_site "default" do action [:delete] end @@ -302,12 +247,11 @@ template "/etc/logrotate.d/nginx" do mode "644" end -# Updates +### Import, update and maintenance scripts %w[nominatim-update - nominatim-update-source - nominatim-update-refresh-db nominatim-update-data + nominatim-update-refresh-db nominatim-daily-maintenance].each do |fname| template "#{bin_directory}/#{fname}" do source "#{fname}.erb" @@ -315,10 +259,8 @@ end group "nominatim" mode "554" variables :bindir => bin_directory, - :srcdir => source_directory, - :builddir => build_directory, :projectdir => project_directory, - :qabindir => qa_bin_directory, + :venvprefix => "#{python_directory}/bin/", :qadatadir => qa_data_directory end end @@ -327,7 +269,7 @@ systemd_service "nominatim-update" do description "Update the Nominatim database" exec_start "#{bin_directory}/nominatim-update" restart "on-success" - standard_output "append:#{node[:nominatim][:logdir]}/update.log" + standard_output "journal" standard_error "inherit" working_directory project_directory end @@ -339,16 +281,16 @@ systemd_service "nominatim-update-maintenance-trigger" do end systemd_timer "nominatim-update-maintenance-trigger" do - action node[:nominatim][:state] != "off" ? :create : :delete + action :create description "Schedule daily maintenance tasks for Nominatim DB" on_calendar "*-*-* 02:03:00 UTC" end service "nominatim-update-maintenance-trigger" do - action node[:nominatim][:state] != "off" ? :enable : :disable + action :enable end -# Nominatim UI +## Nominatim UI git ui_directory do action :sync @@ -365,25 +307,12 @@ template "#{ui_directory}/dist/theme/config.theme.js" do mode "664" end -# Nominatim QA +## Nominatim QA if node[:nominatim][:enable_qa_tiles] - package "python3-geojson" - - git qa_bin_directory do - repository node[:nominatim][:qa_repository] - revision node[:nominatim][:qa_revision] - enable_submodules true - user "nominatim" - group "nominatim" - notifies :run, "execute[compile_qa]" - end - - execute "compile_qa" do - action :nothing - user "nominatim" - cwd "#{qa_bin_directory}/clustering-vt" - command "make" + python_package "nominatim-data-analyser" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] end directory qa_data_directory do @@ -393,7 +322,7 @@ if node[:nominatim][:enable_qa_tiles] recursive true end - template "#{qa_bin_directory}/analyser/config/config.yaml" do + template "#{project_directory}/qa-config.yaml" do source "qa_config.erb" owner "nominatim" group "nominatim" @@ -408,79 +337,12 @@ if node[:nominatim][:enable_qa_tiles] nginx_site "qa-tiles.nominatim" do template "nginx-qa-tiles.erb" - directory build_directory + directory qa_data_directory variables :qa_data_directory => qa_data_directory end - end -# Replication - -cron_d "nominatim-clean-db" do - action node[:nominatim][:state] == "master" ? :create : :delete - minute "5" - hour "*/4" - user "postgres" - command "#{bin_directory}/clean-db-nominatim" - mailto email_errors -end - -if node[:nominatim][:state] == "master" - postgresql_user "replication" do - cluster node[:nominatim][:dbcluster] - password data_bag_item("nominatim", "passwords")["replication"] - replication true - end - - directory node[:rsyncd][:modules][:archive][:path] do - owner "postgres" - group "postgres" - mode "700" - end - - template "#{bin_directory}/clean-db-nominatim" do - source "clean-db-nominatim.erb" - owner "nominatim" - group "nominatim" - mode "755" - variables :archive_dir => node[:rsyncd][:modules][:archive][:path], - :update_stop_file => "#{basedir}/status/updates_disabled", - :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") - end -end - -# Maintenance - -cron_d "nominatim-backup" do - action (node[:nominatim][:enable_backup] && node[:nominatim][:state] != "off") ? :create : :delete - minute "0" - hour "3" - day "1" - user "nominatim" - command "#{bin_directory}/backup-nominatim" - mailto email_errors -end - -cron_d "nominatim-vacuum-db" do - action node[:nominatim][:state] != "off" ? :create : :delete - minute "20" - hour "0" - user "postgres" - command "#{bin_directory}/vacuum-db-nominatim" - mailto email_errors -end - -%w[backup-nominatim vacuum-db-nominatim].each do |fname| - template "#{bin_directory}/#{fname}" do - source "#{fname}.erb" - owner "nominatim" - group "nominatim" - mode "755" - variables :db => node[:nominatim][:dbname] - end -end - -# Logging +## Logging and monitoring template "/etc/logrotate.d/nominatim" do source "logrotate.nominatim.erb" @@ -489,26 +351,6 @@ template "/etc/logrotate.d/nominatim" do mode "644" end -# Monitoring - -munin_plugin_conf "nominatim" do - template "munin.erb" - variables :db => node[:nominatim][:dbname], - :querylog => "#{node[:nominatim][:logdir]}/query.log" -end - -munin_plugin "nominatim_importlag" do - target "#{source_directory}/munin/nominatim_importlag" -end - -munin_plugin "nominatim_query_speed" do - target "#{source_directory}/munin/nominatim_query_speed_querylog" -end - -munin_plugin "nominatim_requests" do - target "#{source_directory}/munin/nominatim_requests_querylog" -end - prometheus_exporter "nominatim" do port 8082 user "www-data" @@ -519,8 +361,6 @@ prometheus_exporter "nominatim" do ] end -include_recipe "fail2ban" - frontend_addresses = frontends.collect { |f| f.ipaddresses(:role => :external) } fail2ban_jail "nominatim_limit_req" do