From 58add207f0be3d805a2022c659954be333a1821d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 15 Nov 2024 10:44:19 +0100 Subject: [PATCH] nominatim: remove all code related to former Ubuntu install --- cookbooks/nominatim/attributes/default.rb | 20 +- cookbooks/nominatim/recipes/default.rb | 1156 ++++------------- .../templates/default/backup-nominatim.erb | 20 - .../default/nominatim-daily-maintenance.erb | 4 - .../default/nominatim-update-refresh-db.erb | 7 - .../default/nominatim-update-source.erb | 24 - .../templates/default/nominatim-update.erb | 45 +- .../templates/default/vacuum-db-nominatim.erb | 8 - roles/nominatim.rb | 14 +- 9 files changed, 278 insertions(+), 1020 deletions(-) delete mode 100644 cookbooks/nominatim/templates/default/backup-nominatim.erb delete mode 100644 cookbooks/nominatim/templates/default/nominatim-update-source.erb delete mode 100644 cookbooks/nominatim/templates/default/vacuum-db-nominatim.erb diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb index a7037ee20..1b82d5f79 100644 --- a/cookbooks/nominatim/attributes/default.rb +++ b/cookbooks/nominatim/attributes/default.rb @@ -1,41 +1,23 @@ -default[:nominatim][:state] = "off" # or: standalone, master, slave default[:nominatim][:dbadmins] = [] default[:nominatim][:dbcluster] = "14/main" default[:nominatim][:dbname] = "nominatim" -default[:nominatim][:tablespaces] = [] default[:nominatim][:postgis] = "3" default[:nominatim][:logdir] = "/var/log/nominatim" default[:nominatim][:pip_index] = "https://nominatim.org/data/osm-production" -default[:nominatim][:repository] = "https://git.openstreetmap.org/public/nominatim.git" -default[:nominatim][:revision] = "deploy" -default[:nominatim][:enable_backup] = false -default[:nominatim][:enable_git_updates] = true default[:nominatim][:enable_qa_tiles] = false default[:nominatim][:ui_repository] = "https://git.openstreetmap.org/public/nominatim-ui.git" default[:nominatim][:ui_revision] = "master" default[:nominatim][:qa_repository] = "https://github.com/osm-search/Nominatim-Data-Analyser" default[:nominatim][:qa_revision] = "deploy" -default[:nominatim][:api_flavour] = "python" default[:nominatim][:api_workers] = 10 default[:nominatim][:api_pool_size] = 10 default[:nominatim][:api_query_timeout] = 5 default[:nominatim][:api_request_timeout] = 20 -default[:nominatim][:fpm_pools] = { - "nominatim.openstreetmap.org" => { - :pm => "dynamic", - :max_children => 60, - :prometheus_port => 9253 - } -} - default[:nominatim][:config] = { - :tokenizer => "icu", - :forward_dependencies => "no" + :tokenizer => "icu" } -default[:nominatim][:redirects] = {} - default[:postgresql][:versions] |= [node[:nominatim][:dbcluster].split("/").first] default[:postgresql][:monitor_database] = "nominatim" diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 78f0ff54d..bc7e542a7 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -19,980 +19,348 @@ include_recipe "accounts" include_recipe "prometheus" +include_recipe "postgresql" +include_recipe "python" +include_recipe "nginx" +include_recipe "git" +include_recipe "fail2ban" + +basedir = data_bag_item("accounts", "nominatim")["home"] +project_directory = "#{basedir}/planet-project" +bin_directory = "#{basedir}/bin" +cfg_directory = "#{basedir}/etc" +ui_directory = "#{basedir}/ui" +qa_bin_directory = "#{basedir}/Nominatim-Data-Analyser" +qa_data_directory = "#{basedir}/qa-data" + +directory basedir do + owner "nominatim" + group "nominatim" + mode "755" + recursive true +end -if platform?("debian") - include_recipe "postgresql" - include_recipe "python" - include_recipe "nginx" - include_recipe "git" - include_recipe "fail2ban" - - basedir = data_bag_item("accounts", "nominatim")["home"] - project_directory = "#{basedir}/planet-project" - bin_directory = "#{basedir}/bin" - cfg_directory = "#{basedir}/etc" - ui_directory = "#{basedir}/ui" - qa_bin_directory = "#{basedir}/Nominatim-Data-Analyser" - qa_data_directory = "#{basedir}/qa-data" - - directory basedir do +[basedir, bin_directory, cfg_directory, project_directory, ui_directory].each do |path| + directory path do owner "nominatim" group "nominatim" mode "755" - recursive true - end - - [basedir, bin_directory, cfg_directory, project_directory, ui_directory].each do |path| - directory path do - owner "nominatim" - group "nominatim" - mode "755" - end - end - - if node[:nominatim][:flatnode_file] - directory File.dirname(node[:nominatim][:flatnode_file]) do - recursive true - end end +end - directory "#{bin_directory}/maintenance" do - owner "nominatim" - group "nominatim" - mode "775" +if node[:nominatim][:flatnode_file] + directory File.dirname(node[:nominatim][:flatnode_file]) do + recursive true end +end - ## Log directory setup +directory "#{bin_directory}/maintenance" do + owner "nominatim" + group "nominatim" + mode "775" +end - directory node[:nominatim][:logdir] do - owner "nominatim" - group "nominatim" - mode "755" - recursive true - end +## Log directory setup - file "#{node[:nominatim][:logdir]}/query.log" do - action :create_if_missing - owner "www-data" - group "adm" - mode "664" - end +directory node[:nominatim][:logdir] do + owner "nominatim" + group "nominatim" + mode "755" + recursive true +end - ### Postgresql +file "#{node[:nominatim][:logdir]}/query.log" do + action :create_if_missing + owner "www-data" + group "adm" + mode "664" +end - postgresql_version = node[:nominatim][:dbcluster].split("/").first - postgis_version = node[:nominatim][:postgis] +### Postgresql - package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" +postgresql_version = node[:nominatim][:dbcluster].split("/").first +postgis_version = node[:nominatim][:postgis] - node[:nominatim][:dbadmins].each do |user| - postgresql_user user do - cluster node[:nominatim][:dbcluster] - superuser true - end - end +package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" - postgresql_user "nominatim" do +node[:nominatim][:dbadmins].each do |user| + postgresql_user user do cluster node[:nominatim][:dbcluster] superuser true end +end - postgresql_user "www-data" do - cluster node[:nominatim][:dbcluster] - end - - ### Nominatim - - python_directory = "#{basedir}/venv" - - package %w[ - build-essential - libicu-dev - python3-dev - pkg-config - osm2pgsql - ruby - ruby-file-tail - ruby-pg - ruby-webrick - ] - - python_virtualenv python_directory do - interpreter "/usr/bin/python3" - end - - # These are updated during the database update. - python_package "nominatim-db" do - python_virtualenv python_directory - extra_index_url node[:nominatim][:pip_index] - end - - python_package "nominatim-api" do - python_virtualenv python_directory - extra_index_url node[:nominatim][:pip_index] - end - - remote_directory "#{project_directory}/static-website" do - source "website" - owner "nominatim" - group "nominatim" - mode "755" - files_owner "nominatim" - files_group "nominatim" - files_mode "644" - purge false - end - - template "#{project_directory}/.env" do - source "nominatim.env.erb" - owner "nominatim" - group "nominatim" - mode "664" - variables :base_url => "nominatim.openstreetmap.org", - :dbname => node[:nominatim][:dbname], - :flatnode_file => node[:nominatim][:flatnode_file], - :log_file => "#{node[:nominatim][:logdir]}/query.log", - :pool_size => node[:nominatim][:api_pool_size], - :query_timeout => node[:nominatim][:api_query_timeout], - :request_timeout => node[:nominatim][:api_request_timeout] - end - - remote_file "#{project_directory}/secondary_importance.sql.gz" do - action :create_if_missing - source "https://nominatim.org/data/wikimedia-secondary-importance.sql.gz" - owner "nominatim" - group "nominatim" - mode "644" - end - - remote_file "#{project_directory}/wikimedia-importance.csv.gz" do - action :create_if_missing - source "https://nominatim.org/data/wikimedia-importance.csv.gz" - owner "nominatim" - group "nominatim" - mode "644" - end - - %w[gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| - remote_file "#{project_directory}/#{fname}" do - action :create - source "https://nominatim.org/data/#{fname}" - owner "nominatim" - group "nominatim" - mode "644" - end - end - - # Webserver + frontend +postgresql_user "nominatim" do + cluster node[:nominatim][:dbcluster] + superuser true +end - %w[user_agent referrer email generic].each do |name| - file "#{cfg_directory}/nginx_blocked_#{name}.conf" do - action :create_if_missing - owner "nominatim" - group "adm" - mode "664" - end - end +postgresql_user "www-data" do + cluster node[:nominatim][:dbcluster] +end - systemd_service "nominatim" do - description "Nominatim running as a gunicorn application" - user "www-data" - group "www-data" - working_directory project_directory - standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log" - standard_error "inherit" - exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker 'nominatim_api.server.falcon.server:run_wsgi()'" - exec_reload "/bin/kill -s HUP $MAINPID" - kill_mode "mixed" - timeout_stop_sec 5 - private_tmp true - requires "nominatim.socket" - after "network.target" - end +### Nominatim - systemd_socket "nominatim" do - description "Gunicorn socket for Nominatim" - listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock" - socket_user "www-data" - end +python_directory = "#{basedir}/venv" - ssl_certificate node[:fqdn] do - domains [node[:fqdn], - "nominatim.openstreetmap.org", - "nominatim.osm.org", - "nominatim.openstreetmap.com", - "nominatim.openstreetmap.net", - "nominatim.openstreetmaps.org", - "nominatim.openmaps.org", - "nominatim.qgis.org"] - notifies :reload, "service[nginx]" - end +package %w[ + build-essential + libicu-dev + python3-dev + pkg-config + osm2pgsql + ruby + ruby-file-tail + ruby-pg + ruby-webrick +] - nginx_site "default" do - action [:delete] - end +python_virtualenv python_directory do + interpreter "/usr/bin/python3" +end - frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) +# These are updated during the database update. +python_package "nominatim-db" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] +end - nginx_site "nominatim" do - template "nginx.erb" - directory project_directory - variables :pools => node[:nominatim][:fpm_pools], - :frontends => frontends, - :confdir => "#{basedir}/etc", - :ui_directory => ui_directory - end +python_package "nominatim-api" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] +end - template "/etc/logrotate.d/nginx" do - source "logrotate.nginx.erb" - owner "root" - group "root" - mode "644" - end +remote_directory "#{project_directory}/static-website" do + source "website" + owner "nominatim" + group "nominatim" + mode "755" + files_owner "nominatim" + files_group "nominatim" + files_mode "644" + purge false +end - ### Import, update and maintenance scripts - - %w[nominatim-update - nominatim-update-data - nominatim-update-refresh-db - nominatim-daily-maintenance].each do |fname| - template "#{bin_directory}/#{fname}" do - source "#{fname}.erb" - owner "nominatim" - group "nominatim" - mode "554" - variables :bindir => bin_directory, - :projectdir => project_directory, - :venvprefix => "#{python_directory}/bin/", - :qadatadir => qa_data_directory - end - end +template "#{project_directory}/.env" do + source "nominatim.env.erb" + owner "nominatim" + group "nominatim" + mode "664" + variables :base_url => "nominatim.openstreetmap.org", + :dbname => node[:nominatim][:dbname], + :flatnode_file => node[:nominatim][:flatnode_file], + :log_file => "#{node[:nominatim][:logdir]}/query.log", + :pool_size => node[:nominatim][:api_pool_size], + :query_timeout => node[:nominatim][:api_query_timeout], + :request_timeout => node[:nominatim][:api_request_timeout] +end - systemd_service "nominatim-update" do - description "Update the Nominatim database" - exec_start "#{bin_directory}/nominatim-update" - restart "on-success" - standard_output "journal" - standard_error "inherit" - working_directory project_directory - end +remote_file "#{project_directory}/secondary_importance.sql.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-secondary-importance.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" +end - systemd_service "nominatim-update-maintenance-trigger" do - description "Trigger daily maintenance tasks for Nominatim DB" - exec_start "ln -sf #{bin_directory}/nominatim-daily-maintenance #{bin_directory}/maintenance/" - user "nominatim" - end +remote_file "#{project_directory}/wikimedia-importance.csv.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-importance.csv.gz" + owner "nominatim" + group "nominatim" + mode "644" +end - systemd_timer "nominatim-update-maintenance-trigger" do +%w[gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| + remote_file "#{project_directory}/#{fname}" do action :create - description "Schedule daily maintenance tasks for Nominatim DB" - on_calendar "*-*-* 02:03:00 UTC" - end - - service "nominatim-update-maintenance-trigger" do - action :enable - end - - ## Nominatim UI - - git ui_directory do - action :sync - repository node[:nominatim][:ui_repository] - revision node[:nominatim][:ui_revision] - user "nominatim" - group "nominatim" - end - - template "#{ui_directory}/dist/theme/config.theme.js" do - source "ui-config.js.erb" + source "https://nominatim.org/data/#{fname}" owner "nominatim" group "nominatim" - mode "664" - end - - ## Nominatim QA - - if node[:nominatim][:enable_qa_tiles] - python_package "nominatim-data-analyser" do - python_virtualenv python_directory - extra_index_url node[:nominatim][:pip_index] - end - - directory qa_data_directory do - owner "nominatim" - group "nominatim" - mode "755" - recursive true - end - - template "#{project_directory}/qa-config.yaml" do - source "qa_config.erb" - owner "nominatim" - group "nominatim" - mode "755" - variables :outputdir => "#{qa_data_directory}/new" - end - - ssl_certificate "qa-tile.nominatim.openstreetmap.org" do - domains ["qa-tile.nominatim.openstreetmap.org"] - notifies :reload, "service[nginx]" - end - - nginx_site "qa-tiles.nominatim" do - template "nginx-qa-tiles.erb" - directory qa_data_directory - variables :qa_data_directory => qa_data_directory - end - end - - ## Logging and monitoring - - template "/etc/logrotate.d/nominatim" do - source "logrotate.nominatim.erb" - owner "root" - group "root" mode "644" end +end - prometheus_exporter "nominatim" do - port 8082 - user "www-data" - restrict_address_families "AF_UNIX" - options [ - "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log", - "--nominatim.database-name=#{node[:nominatim][:dbname]}" - ] - end - -################################# END OF DEBIAN ############################# - -else - - if node[:nominatim][:api_flavour] == "php" - include_recipe "php::fpm" - end - - basedir = data_bag_item("accounts", "nominatim")["home"] - email_errors = data_bag_item("accounts", "lonvia")["email"] - - directory basedir do - owner "nominatim" - group "nominatim" - mode "755" - recursive true - end - - ## Log directory setup - - directory node[:nominatim][:logdir] do - owner "nominatim" - group "nominatim" - mode "755" - recursive true - end - - file "#{node[:nominatim][:logdir]}/query.log" do - action :create_if_missing - owner "www-data" - group "adm" - mode "664" - end +# Webserver + frontend - file "#{node[:nominatim][:logdir]}/update.log" do +%w[user_agent referrer email generic].each do |name| + file "#{cfg_directory}/nginx_blocked_#{name}.conf" do action :create_if_missing owner "nominatim" group "adm" mode "664" end +end - ## Postgresql - - include_recipe "postgresql" - - postgresql_version = node[:nominatim][:dbcluster].split("/").first - postgis_version = node[:nominatim][:postgis] - - package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" - - node[:nominatim][:dbadmins].each do |user| - postgresql_user user do - cluster node[:nominatim][:dbcluster] - superuser true - only_if { node[:nominatim][:state] != "slave" } - end - end +systemd_service "nominatim" do + description "Nominatim running as a gunicorn application" + user "www-data" + group "www-data" + working_directory project_directory + standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log" + standard_error "inherit" + exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker 'nominatim_api.server.falcon.server:run_wsgi()'" + exec_reload "/bin/kill -s HUP $MAINPID" + kill_mode "mixed" + timeout_stop_sec 5 + private_tmp true + requires "nominatim.socket" + after "network.target" +end - postgresql_user "nominatim" do - cluster node[:nominatim][:dbcluster] - superuser true - only_if { node[:nominatim][:state] != "slave" } - end +systemd_socket "nominatim" do + description "Gunicorn socket for Nominatim" + listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock" + socket_user "www-data" +end - postgresql_user "www-data" do - cluster node[:nominatim][:dbcluster] - only_if { node[:nominatim][:state] != "slave" } - end +ssl_certificate node[:fqdn] do + domains [node[:fqdn], + "nominatim.openstreetmap.org", + "nominatim.osm.org", + "nominatim.openstreetmap.com", + "nominatim.openstreetmap.net", + "nominatim.openstreetmaps.org", + "nominatim.openmaps.org", + "nominatim.qgis.org"] + notifies :reload, "service[nginx]" +end - directory "#{basedir}/tablespaces" do - owner "postgres" - group "postgres" - mode "700" - end +nginx_site "default" do + action [:delete] +end - # NOTE: tablespaces must be exactly in the same location on each - # Nominatim instance when replication is in use. Therefore - # use symlinks to canonical directory locations. - node[:nominatim][:tablespaces].each do |name, location| - directory location do - owner "postgres" - group "postgres" - mode "700" - recursive true - end - - link "#{basedir}/tablespaces/#{name}" do - to location - end - - postgresql_tablespace name do - cluster node[:nominatim][:dbcluster] - location "#{basedir}/tablespaces/#{name}" - end - end +frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) - ## Nominatim backend - - include_recipe "git" - include_recipe "python" - - python_directory = "#{basedir}/venv" - - package %w[ - build-essential - cmake - g++ - libboost-dev - libboost-system-dev - libboost-filesystem-dev - libexpat1-dev - zlib1g-dev - libbz2-dev - libpq-dev - libproj-dev - liblua5.3-dev - libluajit-5.1-dev - libicu-dev - nlohmann-json3-dev - lua5.3 - python3-pyosmium - python3-psycopg2 - python3-dotenv - python3-psutil - python3-jinja2 - python3-icu - python3-datrie - python3-yaml - python3-sqlalchemy-ext - python3-geoalchemy2 - python3-asyncpg - python3-dev - pkg-config - ruby - ruby-file-tail - ruby-pg - ruby-webrick - ] +nginx_site "nominatim" do + template "nginx.erb" + directory project_directory + variables :pools => node[:nominatim][:fpm_pools], + :frontends => frontends, + :confdir => "#{basedir}/etc", + :ui_directory => ui_directory +end - if node[:nominatim][:api_flavour] == "php" - package %w[ - php-pgsql - php-intl - ] - elsif node[:nominatim][:api_flavour] == "python" - - python_virtualenv python_directory do - interpreter "/usr/bin/python3" - end - - python_package "SQLAlchemy" do - python_virtualenv python_directory - version "2.0.32" - end - - python_package "PyICU" do - python_virtualenv python_directory - version "2.13.1" - end - - python_package "psycopg[binary]" do - python_virtualenv python_directory - version "3.2.1" - end - - python_package "psycopg2-binary" do - python_virtualenv python_directory - version "2.9.9" - end - - python_package "python-dotenv" do - python_virtualenv python_directory - version "1.0.1" - end - - python_package "pygments" do - python_virtualenv python_directory - version "2.18.0" - end - - python_package "PyYAML" do - python_virtualenv python_directory - version "6.0.2" - end - - python_package "falcon" do - python_virtualenv python_directory - version "3.1.3" - end - - python_package "uvicorn" do - python_virtualenv python_directory - version "0.30.5" - end - - python_package "gunicorn" do - python_virtualenv python_directory - version "22.0.0" - end - - python_package "jinja2" do - python_virtualenv python_directory - version "3.1.4" - end - - python_package "datrie" do - python_virtualenv python_directory - version "0.8.2" - end - - python_package "psutil" do - python_virtualenv python_directory - version "6.0.0" - end - - python_package "osmium" do - python_virtualenv python_directory - version "3.7.0" - end - end +template "/etc/logrotate.d/nginx" do + source "logrotate.nginx.erb" + owner "root" + group "root" + mode "644" +end - source_directory = "#{basedir}/src/nominatim" - build_directory = "#{basedir}/src/build" - project_directory = "#{basedir}/planet-project" - bin_directory = "#{basedir}/bin" - cfg_directory = "#{basedir}/etc" - ui_directory = "#{basedir}/ui" - qa_bin_directory = "#{basedir}/src/Nominatim-Data-Analyser" - qa_data_directory = "#{basedir}/qa-data" - - [basedir, "#{basedir}/src", cfg_directory, bin_directory, build_directory, project_directory].each do |path| - directory path do - owner "nominatim" - group "nominatim" - mode "755" - recursive true - end - end +### Import, update and maintenance scripts - directory "#{bin_directory}/maintenance" do +%w[nominatim-update + nominatim-update-data + nominatim-update-refresh-db + nominatim-daily-maintenance].each do |fname| + template "#{bin_directory}/#{fname}" do + source "#{fname}.erb" owner "nominatim" group "nominatim" - mode "775" + mode "554" + variables :bindir => bin_directory, + :projectdir => project_directory, + :venvprefix => "#{python_directory}/bin/", + :qadatadir => qa_data_directory end +end - if node[:nominatim][:flatnode_file] - directory File.dirname(node[:nominatim][:flatnode_file]) do - recursive true - end - end +systemd_service "nominatim-update" do + description "Update the Nominatim database" + exec_start "#{bin_directory}/nominatim-update" + restart "on-success" + standard_output "journal" + standard_error "inherit" + working_directory project_directory +end - remote_directory "#{project_directory}/static-website" do - source "website" - owner "nominatim" - group "nominatim" - mode "755" - files_owner "nominatim" - files_group "nominatim" - files_mode "644" - purge false - end +systemd_service "nominatim-update-maintenance-trigger" do + description "Trigger daily maintenance tasks for Nominatim DB" + exec_start "ln -sf #{bin_directory}/nominatim-daily-maintenance #{bin_directory}/maintenance/" + user "nominatim" +end - # Normally syncing via chef is a bad idea because syncing might involve - # an update of database functions which should not be done while an update - # is ongoing. Therefore we sync in between update cycles. There is an - # exception for slaves: they get DB function updates from the master, so - # only the source code needs to be updated, which chef may do. - git source_directory do - action node[:nominatim][:state] == "slave" ? :sync : :checkout - repository node[:nominatim][:repository] - revision node[:nominatim][:revision] - enable_submodules true - user "nominatim" - group "nominatim" - not_if { node[:nominatim][:state] != "slave" && File.exist?("#{source_directory}/README.md") } - notifies :run, "execute[compile_nominatim]" - end +systemd_timer "nominatim-update-maintenance-trigger" do + action :create + description "Schedule daily maintenance tasks for Nominatim DB" + on_calendar "*-*-* 02:03:00 UTC" +end - remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do - action :create_if_missing - source "https://nominatim.org/data/country_grid.sql.gz" - owner "nominatim" - group "nominatim" - mode "644" - end +service "nominatim-update-maintenance-trigger" do + action :enable +end - execute "compile_nominatim" do - action :nothing - user "nominatim" - cwd build_directory - command "cmake #{source_directory} && make" - notifies :run, "execute[install_nominatim]" - end +## Nominatim UI - execute "install_nominatim" do - action :nothing - cwd build_directory - command "make install" - end +git ui_directory do + action :sync + repository node[:nominatim][:ui_repository] + revision node[:nominatim][:ui_revision] + user "nominatim" + group "nominatim" +end - # Project directory +template "#{ui_directory}/dist/theme/config.theme.js" do + source "ui-config.js.erb" + owner "nominatim" + group "nominatim" + mode "664" +end - template "#{project_directory}/.env" do - source "nominatim.env.erb" - owner "nominatim" - group "nominatim" - mode "664" - variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", - :dbname => node[:nominatim][:dbname], - :flatnode_file => node[:nominatim][:flatnode_file], - :log_file => "#{node[:nominatim][:logdir]}/query.log", - :tokenizer => node[:nominatim][:config][:tokenizer], - :forward_dependencies => node[:nominatim][:config][:forward_dependencies], - :pool_size => node[:nominatim][:api_pool_size], - :query_timeout => node[:nominatim][:api_query_timeout], - :request_timeout => node[:nominatim][:api_request_timeout] - end +## Nominatim QA - remote_file "#{project_directory}/secondary_importance.sql.gz" do - action :create_if_missing - source "https://nominatim.org/data/wikimedia-secondary-importance.sql.gz" - owner "nominatim" - group "nominatim" - mode "644" +if node[:nominatim][:enable_qa_tiles] + python_package "nominatim-data-analyser" do + python_virtualenv python_directory + extra_index_url node[:nominatim][:pip_index] end - remote_file "#{project_directory}/wikimedia-importance.sql.gz" do - action :create_if_missing - source "https://nominatim.org/data/wikimedia-importance.sql.gz" + directory qa_data_directory do owner "nominatim" group "nominatim" - mode "644" - end - - %w[gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname| - remote_file "#{project_directory}/#{fname}" do - action :create - source "https://nominatim.org/data/#{fname}" - owner "nominatim" - group "nominatim" - mode "644" - end - end - - # Webserver + frontend - - %w[user_agent referrer email generic].each do |name| - file "#{cfg_directory}/nginx_blocked_#{name}.conf" do - action :create_if_missing - owner "nominatim" - group "adm" - mode "664" - end - end - - if node[:nominatim][:api_flavour] == "php" - node[:nominatim][:fpm_pools].each do |name, data| - php_fpm name do - port data[:port] - pm data[:pm] - pm_max_children data[:max_children] - pm_start_servers 20 - pm_min_spare_servers 10 - pm_max_spare_servers 20 - pm_max_requests 10000 - prometheus_port data[:prometheus_port] - end - end - elsif node[:nominatim][:api_flavour] == "python" - systemd_service "nominatim" do - description "Nominatim running as a gunicorn application" - user "www-data" - group "www-data" - working_directory project_directory - standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log" - standard_error "inherit" - exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi" - exec_reload "/bin/kill -s HUP $MAINPID" - environment :PYTHONPATH => "/usr/local/lib/nominatim/lib-python/" - kill_mode "mixed" - timeout_stop_sec 5 - private_tmp true - requires "nominatim.socket" - after "network.target" - end - - systemd_socket "nominatim" do - description "Gunicorn socket for Nominatim" - listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock" - socket_user "www-data" - end - end - - ssl_certificate node[:fqdn] do - domains [node[:fqdn], - "nominatim.openstreetmap.org", - "nominatim.osm.org", - "nominatim.openstreetmap.com", - "nominatim.openstreetmap.net", - "nominatim.openstreetmaps.org", - "nominatim.openmaps.org", - "nominatim.qgis.org"] - notifies :reload, "service[nginx]" - end - - include_recipe "nginx" - - nginx_site "default" do - action [:delete] - end - - frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) - - nginx_site "nominatim" do - template "nginx.erb" - directory project_directory - variables :pools => node[:nominatim][:fpm_pools], - :frontends => frontends, - :confdir => "#{basedir}/etc", - :ui_directory => ui_directory - end - - template "/etc/logrotate.d/nginx" do - source "logrotate.nginx.erb" - owner "root" - group "root" - mode "644" - end - - # Updates - - %w[nominatim-update - nominatim-update-source - nominatim-update-refresh-db - nominatim-update-data - nominatim-daily-maintenance].each do |fname| - template "#{bin_directory}/#{fname}" do - source "#{fname}.erb" - owner "nominatim" - group "nominatim" - mode "554" - variables :bindir => bin_directory, - :srcdir => source_directory, - :builddir => build_directory, - :projectdir => project_directory, - :qabindir => qa_bin_directory, - :qadatadir => qa_data_directory - end - end - - systemd_service "nominatim-update" do - description "Update the Nominatim database" - exec_start "#{bin_directory}/nominatim-update" - restart "on-success" - standard_output "append:#{node[:nominatim][:logdir]}/update.log" - standard_error "inherit" - working_directory project_directory - end - - systemd_service "nominatim-update-maintenance-trigger" do - description "Trigger daily maintenance tasks for Nominatim DB" - exec_start "ln -sf #{bin_directory}/nominatim-daily-maintenance #{bin_directory}/maintenance/" - user "nominatim" - end - - systemd_timer "nominatim-update-maintenance-trigger" do - action node[:nominatim][:state] != "off" ? :create : :delete - description "Schedule daily maintenance tasks for Nominatim DB" - on_calendar "*-*-* 02:03:00 UTC" - end - - service "nominatim-update-maintenance-trigger" do - action node[:nominatim][:state] != "off" ? :enable : :disable - end - - # Nominatim UI - - git ui_directory do - action :sync - repository node[:nominatim][:ui_repository] - revision node[:nominatim][:ui_revision] - user "nominatim" - group "nominatim" + mode "755" + recursive true end - template "#{ui_directory}/dist/theme/config.theme.js" do - source "ui-config.js.erb" + template "#{project_directory}/qa-config.yaml" do + source "qa_config.erb" owner "nominatim" group "nominatim" - mode "664" - end - - # Nominatim QA - - if node[:nominatim][:enable_qa_tiles] - package "python3-geojson" - - git qa_bin_directory do - repository node[:nominatim][:qa_repository] - revision node[:nominatim][:qa_revision] - enable_submodules true - user "nominatim" - group "nominatim" - notifies :run, "execute[compile_qa]" - end - - execute "compile_qa" do - action :nothing - user "nominatim" - cwd "#{qa_bin_directory}/clustering-vt" - command "make" - end - - directory qa_data_directory do - owner "nominatim" - group "nominatim" - mode "755" - recursive true - end - - template "#{qa_bin_directory}/analyser/config/config.yaml" do - source "qa_config.erb" - owner "nominatim" - group "nominatim" - mode "755" - variables :outputdir => "#{qa_data_directory}/new" - end - - ssl_certificate "qa-tile.nominatim.openstreetmap.org" do - domains ["qa-tile.nominatim.openstreetmap.org"] - notifies :reload, "service[nginx]" - end - - nginx_site "qa-tiles.nominatim" do - template "nginx-qa-tiles.erb" - directory build_directory - variables :qa_data_directory => qa_data_directory - end - - end - - # Replication - - cron_d "nominatim-clean-db" do - action node[:nominatim][:state] == "master" ? :create : :delete - minute "5" - hour "*/4" - user "postgres" - command "#{bin_directory}/clean-db-nominatim" - mailto email_errors - end - - if node[:nominatim][:state] == "master" - postgresql_user "replication" do - cluster node[:nominatim][:dbcluster] - password data_bag_item("nominatim", "passwords")["replication"] - replication true - end - - directory node[:rsyncd][:modules][:archive][:path] do - owner "postgres" - group "postgres" - mode "700" - end - - template "#{bin_directory}/clean-db-nominatim" do - source "clean-db-nominatim.erb" - owner "nominatim" - group "nominatim" - mode "755" - variables :archive_dir => node[:rsyncd][:modules][:archive][:path], - :update_stop_file => "#{basedir}/status/updates_disabled", - :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") - end - end - - # Maintenance - - cron_d "nominatim-backup" do - action (node[:nominatim][:enable_backup] && node[:nominatim][:state] != "off") ? :create : :delete - minute "0" - hour "3" - day "1" - user "nominatim" - command "#{bin_directory}/backup-nominatim" - mailto email_errors + mode "755" + variables :outputdir => "#{qa_data_directory}/new" end - cron_d "nominatim-vacuum-db" do - action node[:nominatim][:state] != "off" ? :create : :delete - minute "20" - hour "0" - user "postgres" - command "#{bin_directory}/vacuum-db-nominatim" - mailto email_errors + ssl_certificate "qa-tile.nominatim.openstreetmap.org" do + domains ["qa-tile.nominatim.openstreetmap.org"] + notifies :reload, "service[nginx]" end - %w[backup-nominatim vacuum-db-nominatim].each do |fname| - template "#{bin_directory}/#{fname}" do - source "#{fname}.erb" - owner "nominatim" - group "nominatim" - mode "755" - variables :db => node[:nominatim][:dbname] - end + nginx_site "qa-tiles.nominatim" do + template "nginx-qa-tiles.erb" + directory qa_data_directory + variables :qa_data_directory => qa_data_directory end +end - # Logging - - template "/etc/logrotate.d/nominatim" do - source "logrotate.nominatim.erb" - owner "root" - group "root" - mode "644" - end +## Logging and monitoring - # Monitoring - prometheus_exporter "nominatim" do - port 8082 - user "www-data" - restrict_address_families "AF_UNIX" - options [ - "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log", - "--nominatim.database-name=#{node[:nominatim][:dbname]}" - ] - end +template "/etc/logrotate.d/nominatim" do + source "logrotate.nominatim.erb" + owner "root" + group "root" + mode "644" +end - include_recipe "fail2ban" -end # platform?('debian') +prometheus_exporter "nominatim" do + port 8082 + user "www-data" + restrict_address_families "AF_UNIX" + options [ + "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log", + "--nominatim.database-name=#{node[:nominatim][:dbname]}" + ] +end frontend_addresses = frontends.collect { |f| f.ipaddresses(:role => :external) } diff --git a/cookbooks/nominatim/templates/default/backup-nominatim.erb b/cookbooks/nominatim/templates/default/backup-nominatim.erb deleted file mode 100644 index 757546012..000000000 --- a/cookbooks/nominatim/templates/default/backup-nominatim.erb +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -# DO NOT EDIT - This file is being maintained by Chef - -# Partial read-only backup only, to restore: -# -# ./utils/setup.php --create-db --setup-db --create-functions --create-partition-functions -# pg_restore --no-tablespaces -O -d nominatim $F -# ./utils/setup.php --create-functions --create-partition-functions - - -D=`date +%Y-%m-%d` -F=/tmp/nominatim-${D}.dmp - -pg_dump --file=$F -F c -Z 9 -t file -t country_osm_grid -t '*columns' -t 'import_polygon_*' -t import_status -t place_addressline -t location_property_osmline -t location_postcode -t placex -t search_name -t 'seq_*' -t word <%= @db %> - -export RSYNC_RSH="ssh -ax" -rsync --preallocate --fuzzy $F backup.openstreetmap.org::backup - -rm -f $F diff --git a/cookbooks/nominatim/templates/default/nominatim-daily-maintenance.erb b/cookbooks/nominatim/templates/default/nominatim-daily-maintenance.erb index 03a51318a..60fe84bec 100644 --- a/cookbooks/nominatim/templates/default/nominatim-daily-maintenance.erb +++ b/cookbooks/nominatim/templates/default/nominatim-daily-maintenance.erb @@ -9,11 +9,7 @@ psql -q -d <%= node[:nominatim][:dbname] %> -c 'VACUUM ANALYSE search_name' <% if node[:nominatim][:enable_qa_tiles] -%> pushd <%= @qabindir %> -<% if node.platform?("debian") -%> if <%= @venvprefix %>nominatim-data-analyser --execute-all --config=<%= @projectdir %>/qa-config.yaml; then -<% else -%> -if python3 cli.py --execute-all; then -<% end -%> rm -rf "<%= @qadatadir %>/old" if [ -d "<%= @qadatadir %>/current" ]; then mv "<%= @qadatadir %>/current" "<%= @qadatadir %>/old" diff --git a/cookbooks/nominatim/templates/default/nominatim-update-refresh-db.erb b/cookbooks/nominatim/templates/default/nominatim-update-refresh-db.erb index ff70bea45..7156342e3 100644 --- a/cookbooks/nominatim/templates/default/nominatim-update-refresh-db.erb +++ b/cookbooks/nominatim/templates/default/nominatim-update-refresh-db.erb @@ -6,12 +6,5 @@ date "+%c === Refresh database after software updates" cd <%= @projectdir %> -<% if node.platform?("debian") -%> <%= @venvprefix %>nominatim admin --migrate <%= @venvprefix %>nominatim refresh --functions --address-levels -<% else -%> -# Run the migrations from the nominatim version in the build directory, -# so they can execute while the frontend doesn't have the new code yet. -<%= @builddir %>/nominatim admin --migrate -<%= @builddir %>/nominatim refresh --functions --address-levels --website -<% end -%> diff --git a/cookbooks/nominatim/templates/default/nominatim-update-source.erb b/cookbooks/nominatim/templates/default/nominatim-update-source.erb deleted file mode 100644 index dacb6190e..000000000 --- a/cookbooks/nominatim/templates/default/nominatim-update-source.erb +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -e - -# DO NOT EDIT - This file is being maintained by Chef - -# Note: the script must return 0 only when new updates have been applied. - -date "+%c === Checking for new versions of Nominatim" - -cd <%= @srcdir %> - -git fetch origin -git fetch origin -tf - -if git diff --exit-code <%= node[:nominatim][:revision] %> >/dev/null; then - # signal that there are no new updates - exit 99 -fi - -git checkout --detach <%= node[:nominatim][:revision] %> -git submodule update - -cd <%= @builddir %> -cmake . -make diff --git a/cookbooks/nominatim/templates/default/nominatim-update.erb b/cookbooks/nominatim/templates/default/nominatim-update.erb index e7f8a5f67..470cb7d4a 100644 --- a/cookbooks/nominatim/templates/default/nominatim-update.erb +++ b/cookbooks/nominatim/templates/default/nominatim-update.erb @@ -6,38 +6,21 @@ date "+%c === Starting Nominatim update cycle" starttime=`date +%s` -<% if node.platform?("debian") -%> - EXTRA_PIP='--extra-index-url <%= node[:nominatim][:pip_index] %>' - - # First update the importer and run the migrations. - if <%= @venvprefix %>pip list $EXTRA_PIP --outdated | fgrep -q nominatim-db; then - <%= @venvprefix %>pip install $EXTRA_PIP -U nominatim-db - if ! /sbin/runuser -u nominatim -- <%= @bindir %>/nominatim-update-refresh-db; then - date "+%c === Database refresh failed. Stopping updates." - exit 1 - fi +EXTRA_PIP='--extra-index-url <%= node[:nominatim][:pip_index] %>' + +# First update the importer and run the migrations. +if <%= @venvprefix %>pip list $EXTRA_PIP --outdated | fgrep -q nominatim-db; then + <%= @venvprefix %>pip install $EXTRA_PIP -U nominatim-db + if ! /sbin/runuser -u nominatim -- <%= @bindir %>/nominatim-update-refresh-db; then + date "+%c === Database refresh failed. Stopping updates." + exit 1 fi - # Then only attempt to update the frontend. - if <%= @venvprefix %>pip list $EXTRA_PIP --outdated | fgrep -q nominatim-api; then - <%= @venvprefix %>pip install $EXTRA_PIP -U nominatim-api - systemctl reload nominatim - fi -<% else -%> - <% if node[:nominatim][:enable_git_updates] -%> - if /sbin/runuser -u nominatim -- <%= @bindir %>/nominatim-update-source; then - - if ! /sbin/runuser -u nominatim -- <%= @bindir %>/nominatim-update-refresh-db; then - date "+%c === Database refresh failed. Stopping updates." - exit 1 - fi - - pushd <%= @builddir %> - make install - - systemctl reload nominatim - fi - <% end -%> -<% end -%> +fi +# Then only attempt to update the frontend. +if <%= @venvprefix %>pip list $EXTRA_PIP --outdated | fgrep -q nominatim-api; then + <%= @venvprefix %>pip install $EXTRA_PIP -U nominatim-api + systemctl reload nominatim +fi if ! /sbin/runuser -u nominatim -- <%= @bindir %>/nominatim-update-data; then date "+%c === Data update failed. Stopping updates." diff --git a/cookbooks/nominatim/templates/default/vacuum-db-nominatim.erb b/cookbooks/nominatim/templates/default/vacuum-db-nominatim.erb deleted file mode 100644 index ce027f606..000000000 --- a/cookbooks/nominatim/templates/default/vacuum-db-nominatim.erb +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# -# DO NOT EDIT - This file is being maintained by Chef - - -# Vaccum all tables with indices on integer arrays. -# Agressive vacuuming seems to help against index bloat. -psql -q -d <%= @db %> -c 'VACUUM ANALYSE search_name' diff --git a/roles/nominatim.rb b/roles/nominatim.rb index e79a7fce2..fca0d8ee0 100644 --- a/roles/nominatim.rb +++ b/roles/nominatim.rb @@ -64,19 +64,7 @@ default_attributes( } }, :nominatim => { - :dbadmins => %w[lonvia tomh], - :tablespaces => { - "dosm" => "/ssd/tablespaces/dosm", - "iosm" => "/ssd/tablespaces/iosm", - "dplace" => "/ssd/tablespaces/dplace", - "iplace" => "/ssd/tablespaces/iplace", - "daddress" => "/ssd/tablespaces/daddress", - "iaddress" => "/ssd/tablespaces/iaddress", - "dsearch" => "/ssd/tablespaces/dsearch", - "isearch" => "/ssd/tablespaces/isearch", - "daux" => "/ssd/tablespaces/daux", - "iaux" => "/ssd/tablespaces/iaux" - } + :dbadmins => %w[lonvia tomh] } ) -- 2.39.5