X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/d8664c648f4d2d2c550fa0830cc5e1738a1f17d2..99ff16280f77fdaab1ac5c6f9d87d63413901247:/cookbooks/nominatim/recipes/default.rb?ds=sidebyside diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index a19f203b5..1d0ffe44a 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -18,10 +18,12 @@ # include_recipe "accounts" -include_recipe "munin" -include_recipe "php::fpm" include_recipe "prometheus" +if node[:nominatim][:api_flavour] == "php" + include_recipe "php::fpm" +end + basedir = data_bag_item("accounts", "nominatim")["home"] email_errors = data_bag_item("accounts", "lonvia")["email"] @@ -83,11 +85,6 @@ postgresql_user "www-data" do only_if { node[:nominatim][:state] != "slave" } end -postgresql_munin "nominatim" do - cluster node[:nominatim][:dbcluster] - database node[:nominatim][:dbname] -end - directory "#{basedir}/tablespaces" do owner "postgres" group "postgres" @@ -118,6 +115,9 @@ end ## Nominatim backend include_recipe "git" +include_recipe "python" + +python_directory = "#{basedir}/venv" package %w[ build-essential @@ -133,6 +133,8 @@ package %w[ libproj-dev liblua5.3-dev libluajit-5.1-dev + libicu-dev + nlohmann-json3-dev lua5.3 python3-pyosmium python3-psycopg2 @@ -143,15 +145,98 @@ package %w[ python3-datrie python3-yaml python3-sqlalchemy-ext + python3-geoalchemy2 python3-asyncpg - php-pgsql - php-intl + python3-dev + pkg-config ruby ruby-file-tail ruby-pg ruby-webrick ] +if node[:nominatim][:api_flavour] == "php" + package %w[ + php-pgsql + php-intl + ] +elsif node[:nominatim][:api_flavour] == "python" + + python_virtualenv python_directory do + interpreter "/usr/bin/python3" + end + + python_package "SQLAlchemy" do + python_virtualenv python_directory + version "2.0.32" + end + + python_package "PyICU" do + python_virtualenv python_directory + version "2.13.1" + end + + python_package "psycopg[binary]" do + python_virtualenv python_directory + version "3.2.1" + end + + python_package "psycopg2-binary" do + python_virtualenv python_directory + version "2.9.9" + end + + python_package "python-dotenv" do + python_virtualenv python_directory + version "1.0.1" + end + + python_package "pygments" do + python_virtualenv python_directory + version "2.18.0" + end + + python_package "PyYAML" do + python_virtualenv python_directory + version "6.0.2" + end + + python_package "falcon" do + python_virtualenv python_directory + version "3.1.3" + end + + python_package "uvicorn" do + python_virtualenv python_directory + version "0.30.5" + end + + python_package "gunicorn" do + python_virtualenv python_directory + version "22.0.0" + end + + python_package "jinja2" do + python_virtualenv python_directory + version "3.1.4" + end + + python_package "datrie" do + python_virtualenv python_directory + version "0.8.2" + end + + python_package "psutil" do + python_virtualenv python_directory + version "6.0.0" + end + + python_package "osmium" do + python_virtualenv python_directory + version "3.7.0" + end +end + source_directory = "#{basedir}/src/nominatim" build_directory = "#{basedir}/src/build" project_directory = "#{basedir}/planet-project" @@ -182,6 +267,17 @@ if node[:nominatim][:flatnode_file] end end +remote_directory "#{project_directory}/static-website" do + source "website" + owner "nominatim" + group "nominatim" + mode "755" + files_owner "nominatim" + files_group "nominatim" + files_mode "644" + purge false +end + # Normally syncing via chef is a bad idea because syncing might involve # an update of database functions which should not be done while an update # is ongoing. Therefore we sync in between update cycles. There is an @@ -210,7 +306,7 @@ execute "compile_nominatim" do action :nothing user "nominatim" cwd build_directory - command "cmake -D WITH_LUAJIT=ON #{source_directory} && make" + command "cmake #{source_directory} && make" notifies :run, "execute[install_nominatim]" end @@ -232,7 +328,18 @@ template "#{project_directory}/.env" do :flatnode_file => node[:nominatim][:flatnode_file], :log_file => "#{node[:nominatim][:logdir]}/query.log", :tokenizer => node[:nominatim][:config][:tokenizer], - :forward_dependencies => node[:nominatim][:config][:forward_dependencies] + :forward_dependencies => node[:nominatim][:config][:forward_dependencies], + :pool_size => node[:nominatim][:api_pool_size], + :query_timeout => node[:nominatim][:api_query_timeout], + :request_timeout => node[:nominatim][:api_request_timeout] +end + +remote_file "#{project_directory}/secondary_importance.sql.gz" do + action :create_if_missing + source "https://nominatim.org/data/wikimedia-secondary-importance.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" end remote_file "#{project_directory}/wikimedia-importance.sql.gz" do @@ -264,16 +371,41 @@ end end end -node[:nominatim][:fpm_pools].each do |name, data| - php_fpm name do - port data[:port] - pm data[:pm] - pm_max_children data[:max_children] - pm_start_servers 20 - pm_min_spare_servers 10 - pm_max_spare_servers 20 - pm_max_requests 10000 - prometheus_port data[:prometheus_port] +if node[:nominatim][:api_flavour] == "php" + node[:nominatim][:fpm_pools].each do |name, data| + php_fpm name do + port data[:port] + pm data[:pm] + pm_max_children data[:max_children] + pm_start_servers 20 + pm_min_spare_servers 10 + pm_max_spare_servers 20 + pm_max_requests 10000 + prometheus_port data[:prometheus_port] + end + end +elsif node[:nominatim][:api_flavour] == "python" + systemd_service "nominatim" do + description "Nominatim running as a gunicorn application" + user "www-data" + group "www-data" + working_directory project_directory + standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log" + standard_error "inherit" + exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker nominatim.server.falcon.server:run_wsgi" + exec_reload "/bin/kill -s HUP $MAINPID" + environment :PYTHONPATH => "/usr/local/lib/nominatim/lib-python/" + kill_mode "mixed" + timeout_stop_sec 5 + private_tmp true + requires "nominatim.socket" + after "network.target" + end + + systemd_socket "nominatim" do + description "Gunicorn socket for Nominatim" + listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock" + socket_user "www-data" end end @@ -501,25 +633,6 @@ template "/etc/logrotate.d/nominatim" do end # Monitoring - -munin_plugin_conf "nominatim" do - template "munin.erb" - variables :db => node[:nominatim][:dbname], - :querylog => "#{node[:nominatim][:logdir]}/query.log" -end - -munin_plugin "nominatim_importlag" do - target "#{source_directory}/munin/nominatim_importlag" -end - -munin_plugin "nominatim_query_speed" do - target "#{source_directory}/munin/nominatim_query_speed_querylog" -end - -munin_plugin "nominatim_requests" do - target "#{source_directory}/munin/nominatim_requests_querylog" -end - prometheus_exporter "nominatim" do port 8082 user "www-data"