X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/e5f294d3dc676675b2803915571c28337468c3b3..02326106f95c6cee6974e24603876ddd750889f3:/cookbooks/nominatim/recipes/default.rb?ds=sidebyside diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 6141b65d9..452f3a640 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -1,14 +1,14 @@ # -# Cookbook Name:: nominatim +# Cookbook:: nominatim # Recipe:: base # -# Copyright 2015, OpenStreetMap Foundation +# Copyright:: 2015, OpenStreetMap Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -17,383 +17,985 @@ # limitations under the License. # -basedir = data_bag_item("accounts", "nominatim")["home"] -email_errors = data_bag_item("accounts", "lonvia")["email"] +include_recipe "accounts" +include_recipe "prometheus" + +if platform?("debian") + include_recipe "postgresql" + include_recipe "python" + include_recipe "nginx" + include_recipe "git" + include_recipe "fail2ban" + + basedir = data_bag_item("accounts", "nominatim")["home"] + project_directory = "#{basedir}/planet-project" + bin_directory = "#{basedir}/bin" + cfg_directory = "#{basedir}/etc" + ui_directory = "#{basedir}/ui" + qa_bin_directory = "#{basedir}/Nominatim-Data-Analyser" + qa_data_directory = "#{basedir}/qa-data" + + directory basedir do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end -directory basedir do - owner "nominatim" - group "nominatim" - mode 0o755 - recursive true -end + [basedir, bin_directory, cfg_directory, project_directory, ui_directory].each do |path| + directory path do + owner "nominatim" + group "nominatim" + mode "755" + end + end -directory node[:nominatim][:logdir] do - owner "nominatim" - group "nominatim" - mode 0o755 - recursive true -end + if node[:nominatim][:flatnode_file] + directory File.dirname(node[:nominatim][:flatnode_file]) do + recursive true + end + end -file "#{node[:nominatim][:logdir]}/query.log" do - action :create_if_missing - owner "www-data" - group "adm" - mode 0o664 -end + directory "#{bin_directory}/maintenance" do + owner "nominatim" + group "nominatim" + mode "775" + end -file "#{node[:nominatim][:logdir]}/update.log" do - action :create_if_missing - owner "nominatim" - group "adm" - mode 0o664 -end + ## Log directory setup -## Postgresql + directory node[:nominatim][:logdir] do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end -include_recipe "postgresql" + file "#{node[:nominatim][:logdir]}/query.log" do + action :create_if_missing + owner "www-data" + group "adm" + mode "664" + end -postgresql_version = node[:nominatim][:dbcluster].split("/").first + ### Postgresql -package "postgis" -package "postgresql-#{postgresql_version}-postgis-2.3" + postgresql_version = node[:nominatim][:dbcluster].split("/").first + postgis_version = node[:nominatim][:postgis] + + package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" + + node[:nominatim][:dbadmins].each do |user| + postgresql_user user do + cluster node[:nominatim][:dbcluster] + superuser true + end + end -node[:nominatim][:dbadmins].each do |user| - postgresql_user user do + postgresql_user "nominatim" do cluster node[:nominatim][:dbcluster] superuser true - only_if { node[:nominatim][:state] != "slave" } end -end -postgresql_user "nominatim" do - cluster node[:nominatim][:dbcluster] - superuser true - only_if { node[:nominatim][:state] != "slave" } -end + postgresql_user "www-data" do + cluster node[:nominatim][:dbcluster] + end -postgresql_user "www-data" do - cluster node[:nominatim][:dbcluster] - only_if { node[:nominatim][:state] != "slave" } -end + ### Nominatim -postgresql_munin "nominatim" do - cluster node[:nominatim][:dbcluster] - database node[:nominatim][:dbname] -end + python_directory = "#{basedir}/venv" -directory "#{basedir}/tablespaces" do - owner "postgres" - group "postgres" - mode 0o700 -end + package %w[ + build-essential + libicu-dev + python3-dev + pkg-config + osm2pgsql + ] -# Note: tablespaces must be exactly in the same location on each -# Nominatim instance when replication is in use. Therefore -# use symlinks to canonical directory locations. -node[:nominatim][:tablespaces].each do |name, location| - directory location do - owner "postgres" - group "postgres" - mode 0o700 + python_virtualenv python_directory do + interpreter "/usr/bin/python3" + end + + # These are updated during the database update. + python_package "nominatim-db" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] + end + + python_package "nominatim-api" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] + end + + remote_directory "#{project_directory}/static-website" do + source "website" + owner "nominatim" + group "nominatim" + mode "755" + files_owner "nominatim" + files_group "nominatim" + files_mode "644" + purge false + end + + template "#{project_directory}/.env" do + source "nominatim.env.erb" + owner "nominatim" + group "nominatim" + mode "664" + variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", + :dbname => node[:nominatim][:dbname], + :flatnode_file => node[:nominatim][:flatnode_file], + :log_file => "#{node[:nominatim][:logdir]}/query.log", + :pool_size => node[:nominatim][:api_pool_size], + :query_timeout => node[:nominatim][:api_query_timeout], + :request_timeout => node[:nominatim][:api_request_timeout] + end + + remote_file "#{project_directory}/secondary_importance.sql.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-secondary-importance.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" + end + + remote_file "#{project_directory}/wikimedia-importance.csv.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-importance.csv.gz" + owner "nominatim" + group "nominatim" + mode "644" + end + + %w[gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| + remote_file "#{project_directory}/#{fname}" do + action :create + source "https://nominatim.org/data/#{fname}" + owner "nominatim" + group "nominatim" + mode "644" + end + end + + # Webserver + frontend + + %w[user_agent referrer email generic].each do |name| + file "#{cfg_directory}/nginx_blocked_#{name}.conf" do + action :create_if_missing + owner "nominatim" + group "adm" + mode "664" + end + end + + systemd_service "nominatim" do + description "Nominatim running as a gunicorn application" + user "www-data" + group "www-data" + working_directory project_directory + standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log" + standard_error "inherit" + exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker 'nominatim_api.server.falcon.server:run_wsgi()'" + exec_reload "/bin/kill -s HUP $MAINPID" + kill_mode "mixed" + timeout_stop_sec 5 + private_tmp true + requires "nominatim.socket" + after "network.target" + end + + systemd_socket "nominatim" do + description "Gunicorn socket for Nominatim" + listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock" + socket_user "www-data" + end + + ssl_certificate node[:fqdn] do + domains [node[:fqdn], + "nominatim.openstreetmap.org", + "nominatim.osm.org", + "nominatim.openstreetmap.com", + "nominatim.openstreetmap.net", + "nominatim.openstreetmaps.org", + "nominatim.openmaps.org", + "nominatim.qgis.org"] + notifies :reload, "service[nginx]" + end + + nginx_site "default" do + action [:delete] + end + + frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) + + nginx_site "nominatim" do + template "nginx.erb" + directory project_directory + variables :pools => node[:nominatim][:fpm_pools], + :frontends => frontends, + :confdir => "#{basedir}/etc", + :ui_directory => ui_directory + end + + template "/etc/logrotate.d/nginx" do + source "logrotate.nginx.erb" + owner "root" + group "root" + mode "644" + end + + ### Import, update and maintenance scripts + + %w[nominatim-update + nominatim-update-data + nominatim-update-refresh-db + nominatim-daily-maintenance].each do |fname| + template "#{bin_directory}/#{fname}" do + source "#{fname}.erb" + owner "nominatim" + group "nominatim" + mode "554" + variables :bindir => bin_directory, + :projectdir => project_directory, + :venvprefix => "#{python_directory}/bin/", + :qadatadir => qa_data_directory + end + end + + systemd_service "nominatim-update" do + description "Update the Nominatim database" + exec_start "#{bin_directory}/nominatim-update" + restart "on-success" + standard_output "journal" + standard_error "inherit" + working_directory project_directory + end + + systemd_service "nominatim-update-maintenance-trigger" do + description "Trigger daily maintenance tasks for Nominatim DB" + exec_start "ln -sf #{bin_directory}/nominatim-daily-maintenance #{bin_directory}/maintenance/" + user "nominatim" + end + + systemd_timer "nominatim-update-maintenance-trigger" do + action node[:nominatim][:state] != "off" ? :create : :delete + description "Schedule daily maintenance tasks for Nominatim DB" + on_calendar "*-*-* 02:03:00 UTC" + end + + service "nominatim-update-maintenance-trigger" do + action node[:nominatim][:state] != "off" ? :enable : :disable + end + + ## Nominatim UI + + git ui_directory do + action :sync + repository node[:nominatim][:ui_repository] + revision node[:nominatim][:ui_revision] + user "nominatim" + group "nominatim" + end + + template "#{ui_directory}/dist/theme/config.theme.js" do + source "ui-config.js.erb" + owner "nominatim" + group "nominatim" + mode "664" + end + + ## Nominatim QA + + if node[:nominatim][:enable_qa_tiles] + python_package "nominatim-data-analyser" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] + end + + directory qa_data_directory do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end + + template "#{project_directory}/qa-config.yaml" do + source "qa_config.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :outputdir => "#{qa_data_directory}/new" + end + + ssl_certificate "qa-tile.nominatim.openstreetmap.org" do + domains ["qa-tile.nominatim.openstreetmap.org"] + notifies :reload, "service[nginx]" + end + + nginx_site "qa-tiles.nominatim" do + template "nginx-qa-tiles.erb" + directory qa_data_directory + variables :qa_data_directory => qa_data_directory + end + end + + ## Logging and monitoring + + template "/etc/logrotate.d/nominatim" do + source "logrotate.nominatim.erb" + owner "root" + group "root" + mode "644" + end + + prometheus_exporter "nominatim" do + port 8082 + user "www-data" + restrict_address_families "AF_UNIX" + options [ + "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log", + "--nominatim.database-name=#{node[:nominatim][:dbname]}" + ] + end + +################################# END OF DEBIAN ############################# + +else + + if node[:nominatim][:api_flavour] == "php" + include_recipe "php::fpm" + end + + basedir = data_bag_item("accounts", "nominatim")["home"] + email_errors = data_bag_item("accounts", "lonvia")["email"] + + directory basedir do + owner "nominatim" + group "nominatim" + mode "755" recursive true end - link "#{basedir}/tablespaces/#{name}" do - to location + ## Log directory setup + + directory node[:nominatim][:logdir] do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end + + file "#{node[:nominatim][:logdir]}/query.log" do + action :create_if_missing + owner "www-data" + group "adm" + mode "664" + end + + file "#{node[:nominatim][:logdir]}/update.log" do + action :create_if_missing + owner "nominatim" + group "adm" + mode "664" + end + + ## Postgresql + + include_recipe "postgresql" + + postgresql_version = node[:nominatim][:dbcluster].split("/").first + postgis_version = node[:nominatim][:postgis] + + package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" + + node[:nominatim][:dbadmins].each do |user| + postgresql_user user do + cluster node[:nominatim][:dbcluster] + superuser true + only_if { node[:nominatim][:state] != "slave" } + end end - postgresql_tablespace name do + postgresql_user "nominatim" do cluster node[:nominatim][:dbcluster] - location "#{basedir}/tablespaces/#{name}" + superuser true + only_if { node[:nominatim][:state] != "slave" } end -end -if node[:nominatim][:state] == "master" # ~FC023 - postgresql_user "replication" do + postgresql_user "www-data" do cluster node[:nominatim][:dbcluster] - password data_bag_item("nominatim", "passwords")["replication"] - replication true + only_if { node[:nominatim][:state] != "slave" } end - directory node[:rsyncd][:modules][:archive][:path] do + directory "#{basedir}/tablespaces" do owner "postgres" group "postgres" - mode 0o700 + mode "700" end - template "/usr/local/bin/clean-db-nominatim" do - source "clean-db-nominatim.erb" - owner "root" - group "root" - mode 0o755 - variables :archive_dir => node[:rsyncd][:modules][:archive][:path], - :update_stop_file => "#{basedir}/status/updates_disabled", - :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") + # NOTE: tablespaces must be exactly in the same location on each + # Nominatim instance when replication is in use. Therefore + # use symlinks to canonical directory locations. + node[:nominatim][:tablespaces].each do |name, location| + directory location do + owner "postgres" + group "postgres" + mode "700" + recursive true + end + + link "#{basedir}/tablespaces/#{name}" do + to location + end + + postgresql_tablespace name do + cluster node[:nominatim][:dbcluster] + location "#{basedir}/tablespaces/#{name}" + end end -end -## Nominatim backend - -include_recipe "git" -include_recipe "python" - -package "build-essential" -package "cmake" -package "g++" -package "libboost-dev" -package "libboost-system-dev" -package "libboost-filesystem-dev" -package "libboost-python-dev" -package "libexpat1-dev" -package "zlib1g-dev" -package "libxml2-dev" -package "libbz2-dev" -package "libpq-dev" -package "libgeos++-dev" -package "libproj-dev" -package "osmosis" - -python_package "osmium" - -source_directory = "#{basedir}/nominatim" -build_directory = "#{basedir}/bin" - -directory build_directory do - owner "nominatim" - group "nominatim" - mode 0o755 - recursive true -end + ## Nominatim backend + + include_recipe "git" + include_recipe "python" + + python_directory = "#{basedir}/venv" + + package %w[ + build-essential + cmake + g++ + libboost-dev + libboost-system-dev + libboost-filesystem-dev + libexpat1-dev + zlib1g-dev + libbz2-dev + libpq-dev + libproj-dev + liblua5.3-dev + libluajit-5.1-dev + libicu-dev + nlohmann-json3-dev + lua5.3 + python3-pyosmium + python3-psycopg2 + python3-dotenv + python3-psutil + python3-jinja2 + python3-icu + python3-datrie + python3-yaml + python3-sqlalchemy-ext + python3-geoalchemy2 + python3-asyncpg + python3-dev + pkg-config + ruby + ruby-file-tail + ruby-pg + ruby-webrick + ] + + if node[:nominatim][:api_flavour] == "php" + package %w[ + php-pgsql + php-intl + ] + elsif node[:nominatim][:api_flavour] == "python" + + python_virtualenv python_directory do + interpreter "/usr/bin/python3" + end + + python_package "SQLAlchemy" do + python_virtualenv python_directory + version "2.0.32" + end + + python_package "PyICU" do + python_virtualenv python_directory + version "2.13.1" + end + + python_package "psycopg[binary]" do + python_virtualenv python_directory + version "3.2.1" + end + + python_package "psycopg2-binary" do + python_virtualenv python_directory + version "2.9.9" + end + + python_package "python-dotenv" do + python_virtualenv python_directory + version "1.0.1" + end + + python_package "pygments" do + python_virtualenv python_directory + version "2.18.0" + end + + python_package "PyYAML" do + python_virtualenv python_directory + version "6.0.2" + end + + python_package "falcon" do + python_virtualenv python_directory + version "3.1.3" + end + + python_package "uvicorn" do + python_virtualenv python_directory + version "0.30.5" + end + + python_package "gunicorn" do + python_virtualenv python_directory + version "22.0.0" + end + + python_package "jinja2" do + python_virtualenv python_directory + version "3.1.4" + end + + python_package "datrie" do + python_virtualenv python_directory + version "0.8.2" + end + + python_package "psutil" do + python_virtualenv python_directory + version "6.0.0" + end + + python_package "osmium" do + python_virtualenv python_directory + version "3.7.0" + end + end -# Normally syncing via chef is a bad idea because syncing might involve -# an update of database functions which should not be done while an update -# is ongoing. Therefore we sync in between update cycles. There is an -# exception for slaves: they get DB function updates from the master, so -# only the source code needs to be updated, which chef may do. -git source_directory do - action node[:nominatim][:state] == "slave" ? :sync : :checkout - repository node[:nominatim][:repository] - revision node[:nominatim][:revision] - enable_submodules true - user "nominatim" - group "nominatim" - not_if { node[:nominatim][:state] != "slave" && File.exist?("#{source_directory}/README.md") } - notifies :run, "execute[compile_nominatim]", :immediately -end + source_directory = "#{basedir}/src/nominatim" + build_directory = "#{basedir}/src/build" + project_directory = "#{basedir}/planet-project" + bin_directory = "#{basedir}/bin" + cfg_directory = "#{basedir}/etc" + ui_directory = "#{basedir}/ui" + qa_bin_directory = "#{basedir}/src/Nominatim-Data-Analyser" + qa_data_directory = "#{basedir}/qa-data" + + [basedir, "#{basedir}/src", cfg_directory, bin_directory, build_directory, project_directory].each do |path| + directory path do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end + end -execute "compile_nominatim" do - action :nothing - user "nominatim" - cwd build_directory - command "cmake #{source_directory} && make" -end + directory "#{bin_directory}/maintenance" do + owner "nominatim" + group "nominatim" + mode "775" + end -template "#{source_directory}/.git/hooks/post-merge" do - source "git-post-merge-hook.erb" - owner "nominatim" - group "nominatim" - mode 0o755 - variables :srcdir => source_directory, - :builddir => build_directory, - :dbname => node[:nominatim][:dbname] -end + if node[:nominatim][:flatnode_file] + directory File.dirname(node[:nominatim][:flatnode_file]) do + recursive true + end + end -template "#{build_directory}/settings/local.php" do - source "settings.erb" - owner "nominatim" - group "nominatim" - mode 0o664 - variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", - :dbname => node[:nominatim][:dbname], - :flatnode_file => node[:nominatim][:flatnode_file], - :log_file => "#{node[:nominatim][:logdir]}/query.log" -end + remote_directory "#{project_directory}/static-website" do + source "website" + owner "nominatim" + group "nominatim" + mode "755" + files_owner "nominatim" + files_group "nominatim" + files_mode "644" + purge false + end -if node[:nominatim][:flatnode_file] # ~FC023 - directory File.dirname(node[:nominatim][:flatnode_file]) do - recursive true + # Normally syncing via chef is a bad idea because syncing might involve + # an update of database functions which should not be done while an update + # is ongoing. Therefore we sync in between update cycles. There is an + # exception for slaves: they get DB function updates from the master, so + # only the source code needs to be updated, which chef may do. + git source_directory do + action node[:nominatim][:state] == "slave" ? :sync : :checkout + repository node[:nominatim][:repository] + revision node[:nominatim][:revision] + enable_submodules true + user "nominatim" + group "nominatim" + not_if { node[:nominatim][:state] != "slave" && File.exist?("#{source_directory}/README.md") } + notifies :run, "execute[compile_nominatim]" end -end -template "/etc/logrotate.d/nominatim" do - source "logrotate.nominatim.erb" - owner "root" - group "root" - mode 0o644 -end + remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do + action :create_if_missing + source "https://nominatim.org/data/country_grid.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" + end + + execute "compile_nominatim" do + action :nothing + user "nominatim" + cwd build_directory + command "cmake #{source_directory} && make" + notifies :run, "execute[install_nominatim]" + end -external_data = [ - "wikipedia_article.sql.bin", - "wikipedia_redirect.sql.bin", - "gb_postcode_data.sql.gz" -] + execute "install_nominatim" do + action :nothing + cwd build_directory + command "make install" + end + + # Project directory + + template "#{project_directory}/.env" do + source "nominatim.env.erb" + owner "nominatim" + group "nominatim" + mode "664" + variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", + :dbname => node[:nominatim][:dbname], + :flatnode_file => node[:nominatim][:flatnode_file], + :log_file => "#{node[:nominatim][:logdir]}/query.log", + :tokenizer => node[:nominatim][:config][:tokenizer], + :forward_dependencies => node[:nominatim][:config][:forward_dependencies], + :pool_size => node[:nominatim][:api_pool_size], + :query_timeout => node[:nominatim][:api_query_timeout], + :request_timeout => node[:nominatim][:api_request_timeout] + end -external_data.each do |fname| - remote_file "#{source_directory}/data/#{fname}" do + remote_file "#{project_directory}/secondary_importance.sql.gz" do action :create_if_missing - source "http://www.nominatim.org/data/#{fname}" + source "https://nominatim.org/data/wikimedia-secondary-importance.sql.gz" owner "nominatim" group "nominatim" - mode 0o644 + mode "644" end -end -remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do - action :create_if_missing - source "http://www.nominatim.org/data/country_grid.sql.gz" - owner "nominatim" - group "nominatim" - mode 0o644 -end + remote_file "#{project_directory}/wikimedia-importance.sql.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-importance.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" + end -template "/etc/cron.d/nominatim" do - action node[:nominatim][:state] == :off ? :delete : :create - source "nominatim.cron.erb" - owner "root" - group "root" - mode "0644" - variables :bin_directory => "#{source_directory}/utils", - :mailto => email_errors, - :update_maintenance_trigger => "#{basedir}/status/update_maintenance" -end + %w[gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| + remote_file "#{project_directory}/#{fname}" do + action :create + source "https://nominatim.org/data/#{fname}" + owner "nominatim" + group "nominatim" + mode "644" + end + end -template "#{source_directory}/utils/nominatim-update" do - source "updater.erb" - user "nominatim" - group "nominatim" - mode 0o755 - variables :bindir => build_directory, - :srcdir => source_directory, - :logfile => "#{node[:nominatim][:logdir]}/update.log", - :branch => node[:nominatim][:revision], - :update_stop_file => "#{basedir}/status/updates_disabled", - :update_maintenance_trigger => "#{basedir}/status/update_maintenance" -end + # Webserver + frontend -template "/etc/init.d/nominatim-update" do - source "updater.init.erb" - user "nominatim" - group "nominatim" - mode 0o755 - variables :source_directory => source_directory -end + %w[user_agent referrer email generic].each do |name| + file "#{cfg_directory}/nginx_blocked_#{name}.conf" do + action :create_if_missing + owner "nominatim" + group "adm" + mode "664" + end + end + + if node[:nominatim][:api_flavour] == "php" + node[:nominatim][:fpm_pools].each do |name, data| + php_fpm name do + port data[:port] + pm data[:pm] + pm_max_children data[:max_children] + pm_start_servers 20 + pm_min_spare_servers 10 + pm_max_spare_servers 20 + pm_max_requests 10000 + prometheus_port data[:prometheus_port] + end + end + elsif node[:nominatim][:api_flavour] == "python" + systemd_service "nominatim" do + description "Nominatim running as a gunicorn application" + user "www-data" + group "www-data" + working_directory project_directory + standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log" + standard_error "inherit" + exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi" + exec_reload "/bin/kill -s HUP $MAINPID" + environment :PYTHONPATH => "/usr/local/lib/nominatim/lib-python/" + kill_mode "mixed" + timeout_stop_sec 5 + private_tmp true + requires "nominatim.socket" + after "network.target" + end + + systemd_socket "nominatim" do + description "Gunicorn socket for Nominatim" + listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock" + socket_user "www-data" + end + end + + ssl_certificate node[:fqdn] do + domains [node[:fqdn], + "nominatim.openstreetmap.org", + "nominatim.osm.org", + "nominatim.openstreetmap.com", + "nominatim.openstreetmap.net", + "nominatim.openstreetmaps.org", + "nominatim.openmaps.org", + "nominatim.qgis.org"] + notifies :reload, "service[nginx]" + end + + include_recipe "nginx" + + nginx_site "default" do + action [:delete] + end + + frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) -%w[backup-nominatim vacuum-db-nominatim].each do |fname| - template "/usr/local/bin/#{fname}" do - source "#{fname}.erb" + nginx_site "nominatim" do + template "nginx.erb" + directory project_directory + variables :pools => node[:nominatim][:fpm_pools], + :frontends => frontends, + :confdir => "#{basedir}/etc", + :ui_directory => ui_directory + end + + template "/etc/logrotate.d/nginx" do + source "logrotate.nginx.erb" owner "root" group "root" - mode 0o755 - variables :db => node[:nominatim][:dbname] + mode "644" end -end -## webserver frontend + # Updates + + %w[nominatim-update + nominatim-update-source + nominatim-update-refresh-db + nominatim-update-data + nominatim-daily-maintenance].each do |fname| + template "#{bin_directory}/#{fname}" do + source "#{fname}.erb" + owner "nominatim" + group "nominatim" + mode "554" + variables :bindir => bin_directory, + :srcdir => source_directory, + :builddir => build_directory, + :projectdir => project_directory, + :qabindir => qa_bin_directory, + :qadatadir => qa_data_directory + end + end -template "#{build_directory}/settings/ip_blocks.conf" do - action :create_if_missing - source "ipblocks.erb" - owner "nominatim" - group "nominatim" - mode 0o664 -end + systemd_service "nominatim-update" do + description "Update the Nominatim database" + exec_start "#{bin_directory}/nominatim-update" + restart "on-success" + standard_output "append:#{node[:nominatim][:logdir]}/update.log" + standard_error "inherit" + working_directory project_directory + end -file "#{build_directory}/settings/apache_blocks.conf" do - action :create_if_missing - owner "nominatim" - group "nominatim" - mode 0o664 -end + systemd_service "nominatim-update-maintenance-trigger" do + description "Trigger daily maintenance tasks for Nominatim DB" + exec_start "ln -sf #{bin_directory}/nominatim-daily-maintenance #{bin_directory}/maintenance/" + user "nominatim" + end -file "#{build_directory}/settings/ip_blocks.map" do - action :create_if_missing - owner "nominatim" - group "nominatim" - mode 0o664 -end + systemd_timer "nominatim-update-maintenance-trigger" do + action node[:nominatim][:state] != "off" ? :create : :delete + description "Schedule daily maintenance tasks for Nominatim DB" + on_calendar "*-*-* 02:03:00 UTC" + end -include_recipe "apache" + service "nominatim-update-maintenance-trigger" do + action node[:nominatim][:state] != "off" ? :enable : :disable + end -package "php" -package "php-fpm" -package "php-pgsql" -package "php-pear" -package "php-db" -package "php-intl" + # Nominatim UI -apache_module "rewrite" -apache_module "proxy" -apache_module "proxy_fcgi" -apache_module "proxy_http" -apache_module "headers" + git ui_directory do + action :sync + repository node[:nominatim][:ui_repository] + revision node[:nominatim][:ui_revision] + user "nominatim" + group "nominatim" + end -service "php7.0-fpm" do - action [:enable, :start] - supports :status => true, :restart => true, :reload => true -end + template "#{ui_directory}/dist/theme/config.theme.js" do + source "ui-config.js.erb" + owner "nominatim" + group "nominatim" + mode "664" + end + + # Nominatim QA + + if node[:nominatim][:enable_qa_tiles] + package "python3-geojson" + + git qa_bin_directory do + repository node[:nominatim][:qa_repository] + revision node[:nominatim][:qa_revision] + enable_submodules true + user "nominatim" + group "nominatim" + notifies :run, "execute[compile_qa]" + end + + execute "compile_qa" do + action :nothing + user "nominatim" + cwd "#{qa_bin_directory}/clustering-vt" + command "make" + end + + directory qa_data_directory do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end + + template "#{qa_bin_directory}/analyser/config/config.yaml" do + source "qa_config.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :outputdir => "#{qa_data_directory}/new" + end + + ssl_certificate "qa-tile.nominatim.openstreetmap.org" do + domains ["qa-tile.nominatim.openstreetmap.org"] + notifies :reload, "service[nginx]" + end + + nginx_site "qa-tiles.nominatim" do + template "nginx-qa-tiles.erb" + directory build_directory + variables :qa_data_directory => qa_data_directory + end -node[:nominatim][:fpm_pools].each do |name, data| - template "/etc/php/7.0/fpm/pool.d/#{name}.conf" do - source "fpm.conf.erb" - owner "root" - group "root" - mode 0o644 - variables data.merge(:name => name) - notifies :reload, "service[php7.0-fpm]" end -end -ssl_certificate "nominatim.openstreetmap.org" do - domains ["nominatim.openstreetmap.org", - "nominatim.osm.org", - "nominatim.openstreetmap.com", - "nominatim.openstreetmap.net", - "nominatim.openstreetmaps.org", - "nominatim.openmaps.org"] - notifies :reload, "service[apache2]" -end + # Replication -apache_site "nominatim.openstreetmap.org" do - template "apache.erb" - directory build_directory - variables :pools => node[:nominatim][:fpm_pools] -end + cron_d "nominatim-clean-db" do + action node[:nominatim][:state] == "master" ? :create : :delete + minute "5" + hour "*/4" + user "postgres" + command "#{bin_directory}/clean-db-nominatim" + mailto email_errors + end -apache_site "default" do - action [:disable] -end + if node[:nominatim][:state] == "master" + postgresql_user "replication" do + cluster node[:nominatim][:dbcluster] + password data_bag_item("nominatim", "passwords")["replication"] + replication true + end + + directory node[:rsyncd][:modules][:archive][:path] do + owner "postgres" + group "postgres" + mode "700" + end + + template "#{bin_directory}/clean-db-nominatim" do + source "clean-db-nominatim.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :archive_dir => node[:rsyncd][:modules][:archive][:path], + :update_stop_file => "#{basedir}/status/updates_disabled", + :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") + end + end -template "/etc/logrotate.d/apache2" do - source "logrotate.apache.erb" - owner "root" - group "root" - mode 0o644 -end + # Maintenance -munin_plugin_conf "nominatim" do - template_source "munin.erb" - variables :db => node[:nominatim][:dbname], - :querylog => "#{node[:nominatim][:logdir]}/query.log" -end + cron_d "nominatim-backup" do + action (node[:nominatim][:enable_backup] && node[:nominatim][:state] != "off") ? :create : :delete + minute "0" + hour "3" + day "1" + user "nominatim" + command "#{bin_directory}/backup-nominatim" + mailto email_errors + end -munin_plugin "nominatim_importlag" do - target "#{source_directory}/munin/nominatim_importlag" -end + cron_d "nominatim-vacuum-db" do + action node[:nominatim][:state] != "off" ? :create : :delete + minute "20" + hour "0" + user "postgres" + command "#{bin_directory}/vacuum-db-nominatim" + mailto email_errors + end -munin_plugin "nominatim_query_speed" do - target "#{source_directory}/munin/nominatim_query_speed_querylog" -end + %w[backup-nominatim vacuum-db-nominatim].each do |fname| + template "#{bin_directory}/#{fname}" do + source "#{fname}.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :db => node[:nominatim][:dbname] + end + end -munin_plugin "nominatim_requests" do - target "#{source_directory}/munin/nominatim_requests_querylog" -end + # Logging -munin_plugin "nominatim_throttled_ips" do - target "#{source_directory}/munin/nominatim_throttled_ips" -end + template "/etc/logrotate.d/nominatim" do + source "logrotate.nominatim.erb" + owner "root" + group "root" + mode "644" + end + + # Monitoring + prometheus_exporter "nominatim" do + port 8082 + user "www-data" + restrict_address_families "AF_UNIX" + options [ + "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log", + "--nominatim.database-name=#{node[:nominatim][:dbname]}" + ] + end + + include_recipe "fail2ban" +end # platform?('debian') + +frontend_addresses = frontends.collect { |f| f.ipaddresses(:role => :external) } -directory "#{basedir}/status" do - owner "nominatim" - group "postgres" - mode 0o775 +fail2ban_jail "nominatim_limit_req" do + filter "nginx-limit-req" + logpath "#{node[:nominatim][:logdir]}/nominatim.openstreetmap.org-error.log" + ports [80, 443] + maxretry 20 + ignoreips frontend_addresses.flatten.sort end