X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/60b19818712842f23e5039de2202f7ee45dbb614..874fb8745063242272f3bcccc6de9829f2336583:/cookbooks/nominatim/recipes/default.rb?ds=sidebyside diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 5e1f19d60..571c8fd32 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -20,6 +20,7 @@ include_recipe "accounts" include_recipe "munin" include_recipe "php::fpm" +include_recipe "prometheus" basedir = data_bag_item("accounts", "nominatim")["home"] email_errors = data_bag_item("accounts", "lonvia")["email"] @@ -27,14 +28,14 @@ email_errors = data_bag_item("accounts", "lonvia")["email"] directory basedir do owner "nominatim" group "nominatim" - mode 0o755 + mode "755" recursive true end directory node[:nominatim][:logdir] do owner "nominatim" group "nominatim" - mode 0o755 + mode "755" recursive true end @@ -42,25 +43,14 @@ file "#{node[:nominatim][:logdir]}/query.log" do action :create_if_missing owner "www-data" group "adm" - mode 0o664 + mode "664" end file "#{node[:nominatim][:logdir]}/update.log" do action :create_if_missing owner "nominatim" group "adm" - mode 0o664 -end - -# exception granted for a limited time so that they can set up their own server -firewall_rule "increase-limits-gnome-proxy" do - action :accept - family "inet" - source "net:8.43.85.23" - dest "fw" - proto "tcp:syn" - dest_ports "https" - rate_limit "s:10/sec:30" + mode "664" end ## Postgresql @@ -99,17 +89,17 @@ end directory "#{basedir}/tablespaces" do owner "postgres" group "postgres" - mode 0o700 + mode "700" end -# Note: tablespaces must be exactly in the same location on each +# NOTE: tablespaces must be exactly in the same location on each # Nominatim instance when replication is in use. Therefore # use symlinks to canonical directory locations. node[:nominatim][:tablespaces].each do |name, location| directory location do owner "postgres" group "postgres" - mode 0o700 + mode "700" recursive true end @@ -133,14 +123,14 @@ if node[:nominatim][:state] == "master" directory node[:rsyncd][:modules][:archive][:path] do owner "postgres" group "postgres" - mode 0o700 + mode "700" end template "/usr/local/bin/clean-db-nominatim" do source "clean-db-nominatim.erb" owner "root" group "root" - mode 0o755 + mode "755" variables :archive_dir => node[:rsyncd][:modules][:archive][:path], :update_stop_file => "#{basedir}/status/updates_disabled", :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fqdn] }.join(" ") @@ -168,17 +158,29 @@ package %w[ python3-pyosmium pyosmium python3-psycopg2 + python3-dotenv + python3-psutil + python3-jinja2 + python3-icu + python3-datrie php-pgsql php-intl + php-symfony-dotenv + ruby + ruby-file-tail + ruby-pg ] source_directory = "#{basedir}/nominatim" build_directory = "#{basedir}/bin" +ui_directory = "#{basedir}/ui" +qa_bin_directory = "#{basedir}/Nominatim-Data-Analyser" +qa_data_directory = "#{basedir}/qa-data" directory build_directory do owner "nominatim" group "nominatim" - mode 0o755 + mode "755" recursive true end @@ -195,7 +197,15 @@ git source_directory do user "nominatim" group "nominatim" not_if { node[:nominatim][:state] != "slave" && File.exist?("#{source_directory}/README.md") } - notifies :run, "execute[compile_nominatim]", :immediately + notifies :run, "execute[compile_nominatim]" +end + +remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do + action :create_if_missing + source "https://www.nominatim.org/data/country_grid.sql.gz" + owner "nominatim" + group "nominatim" + mode "644" end execute "compile_nominatim" do @@ -209,21 +219,37 @@ template "#{source_directory}/.git/hooks/post-merge" do source "git-post-merge-hook.erb" owner "nominatim" group "nominatim" - mode 0o755 + mode "755" variables :srcdir => source_directory, :builddir => build_directory, :dbname => node[:nominatim][:dbname] end -template "#{build_directory}/settings/local.php" do - source "settings.erb" +template "#{build_directory}/.env" do + source "nominatim.env.erb" owner "nominatim" group "nominatim" - mode 0o664 + mode "664" variables :base_url => node[:nominatim][:state] == "off" ? node[:fqdn] : "nominatim.openstreetmap.org", :dbname => node[:nominatim][:dbname], :flatnode_file => node[:nominatim][:flatnode_file], - :log_file => "#{node[:nominatim][:logdir]}/query.log" + :log_file => "#{node[:nominatim][:logdir]}/query.log", + :tokenizer => node[:nominatim][:config][:tokenizer] +end + +git ui_directory do + action :sync + repository node[:nominatim][:ui_repository] + revision node[:nominatim][:ui_revision] + user "nominatim" + group "nominatim" +end + +template "#{ui_directory}/dist/theme/config.theme.js" do + source "ui-config.js.erb" + owner "nominatim" + group "nominatim" + mode "664" end if node[:nominatim][:flatnode_file] @@ -236,61 +262,106 @@ template "/etc/logrotate.d/nominatim" do source "logrotate.nominatim.erb" owner "root" group "root" - mode 0o644 + mode "644" end external_data = [ "wikimedia-importance.sql.gz", - "gb_postcode_data.sql.gz" + "gb_postcodes.csv.gz", + "us_postcodes.csv.gz" ] external_data.each do |fname| - remote_file "#{source_directory}/data/#{fname}" do - action :create_if_missing + remote_file "#{build_directory}/#{fname}" do + action :create source "https://www.nominatim.org/data/#{fname}" owner "nominatim" group "nominatim" - mode 0o644 + mode "644" end end -remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do - action :create_if_missing - source "https://www.nominatim.org/data/country_grid.sql.gz" - owner "nominatim" - group "nominatim" - mode 0o644 -end +if node[:nominatim][:state] == "off" + cron_d "nominatim-backup" do + action :delete + end -template "/etc/cron.d/nominatim" do - action node[:nominatim][:state] == "off" ? :delete : :create - source "nominatim.cron.erb" - owner "root" - group "root" - mode "0644" - variables :bin_directory => "#{source_directory}/utils", - :mailto => email_errors, - :update_maintenance_trigger => "#{basedir}/status/update_maintenance" + cron_d "nominatim-vacuum-db" do + action :delete + end + + cron_d "nominatim-clean-db" do + action :delete + end + + systemd_timer "nominatim-update-maintenance-trigger" do + action :delete + end +else + cron_d "nominatim-backup" do + action node[:nominatim][:enable_backup] ? :create : :delete + minute "0" + hour "3" + day "1" + user "nominatim" + command "/usr/local/bin/backup-nominatim" + mailto email_errors + end + + cron_d "nominatim-vacuum-db" do + minute "20" + hour "0" + user "postgres" + command "/usr/local/bin/vacuum-db-nominatim" + mailto email_errors + end + + cron_d "nominatim-clean-db" do + action node[:nominatim][:state] == "master" ? :create : :delete + minute "5" + hour "*/4" + user "postgres" + command "/usr/local/bin/clean-db-nominatim" + mailto email_errors + end + + systemd_service "nominatim-update-maintenance-trigger" do + description "Trigger maintenance tasks for Nominatim DB" + exec_start "touch #{basedir}/status/update_maintenance" + user "nominatim" + end + + systemd_timer "nominatim-update-maintenance-trigger" do + action :create + description "Schedule maintenance tasks for Nominatim DB" + on_calendar "*-*-* 02:03:00 UTC" + end + + service "nominatim-update-maintenance-trigger" do + action [:enable] + end end template "#{source_directory}/utils/nominatim-update" do source "updater.erb" user "nominatim" group "nominatim" - mode 0o755 + mode "755" variables :bindir => build_directory, :srcdir => source_directory, :logfile => "#{node[:nominatim][:logdir]}/update.log", :branch => node[:nominatim][:revision], :update_stop_file => "#{basedir}/status/updates_disabled", - :update_maintenance_trigger => "#{basedir}/status/update_maintenance" + :update_maintenance_trigger => "#{basedir}/status/update_maintenance", + :qabindir => qa_bin_directory, + :qadatadir => qa_data_directory end template "/etc/init.d/nominatim-update" do source "updater.init.erb" user "nominatim" group "nominatim" - mode 0o755 + mode "755" variables :source_directory => source_directory end @@ -299,7 +370,7 @@ end source "#{fname}.erb" owner "root" group "root" - mode 0o755 + mode "755" variables :db => node[:nominatim][:dbname] end end @@ -309,22 +380,28 @@ end directory "#{basedir}/etc" do owner "nominatim" group "adm" - mode 0o775 + mode "775" end -%w[user_agent referrer email].each do |name| +%w[user_agent referrer email generic].each do |name| file "#{basedir}/etc/nginx_blocked_#{name}.conf" do action :create_if_missing owner "nominatim" group "adm" - mode 0o664 + mode "664" end end node[:nominatim][:fpm_pools].each do |name, data| php_fpm name do - template "fpm.conf.erb" - variables data.merge(:name => name) + port data[:port] + pm data[:pm] + pm_max_children data[:max_children] + pm_start_servers 20 + pm_min_spare_servers 10 + pm_max_spare_servers 20 + pm_max_requests 10000 + prometheus_port data[:prometheus_port] end end @@ -335,33 +412,33 @@ ssl_certificate node[:fqdn] do "nominatim.openstreetmap.com", "nominatim.openstreetmap.net", "nominatim.openstreetmaps.org", - "nominatim.openmaps.org"] + "nominatim.openmaps.org", + "nominatim.qgis.org"] notifies :reload, "service[nginx]" end -package "apache2" do - action :remove -end - include_recipe "nginx" nginx_site "default" do action [:delete] end +frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) + nginx_site "nominatim" do template "nginx.erb" directory build_directory variables :pools => node[:nominatim][:fpm_pools], - :frontends => search(:node, "recipes:web\\:\\:frontend"), - :confdir => "#{basedir}/etc" + :frontends => frontends, + :confdir => "#{basedir}/etc", + :ui_directory => ui_directory end template "/etc/logrotate.d/nginx" do source "logrotate.nginx.erb" owner "root" group "root" - mode 0o644 + mode "644" end munin_plugin_conf "nominatim" do @@ -382,17 +459,70 @@ munin_plugin "nominatim_requests" do target "#{source_directory}/munin/nominatim_requests_querylog" end +prometheus_exporter "nominatim" do + port 8082 + user "www-data" + options [ + "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log", + "--nominatim.database-name=#{node[:nominatim][:dbname]}" + ] +end + directory "#{basedir}/status" do owner "nominatim" group "postgres" - mode 0o775 + mode "775" end include_recipe "fail2ban" +frontend_addresses = frontends.collect { |f| f.ipaddresses(:role => :external) } + fail2ban_jail "nominatim_limit_req" do filter "nginx-limit-req" logpath "#{node[:nominatim][:logdir]}/nominatim.openstreetmap.org-error.log" ports [80, 443] - maxretry 5 + maxretry 20 + ignoreips frontend_addresses.flatten.sort +end + +### QA tile generation + +if node[:nominatim][:enable_qa_tiles] + git qa_bin_directory do + repository node[:nominatim][:qa_repository] + revision node[:nominatim][:qa_revision] + enable_submodules true + user "nominatim" + group "nominatim" + notifies :run, "execute[compile_qa]" + end + + execute "compile_qa" do + action :nothing + user "nominatim" + cwd "#{qa_bin_directory}/clustering-vt" + command "make" + end + + directory qa_data_directory do + owner "nominatim" + group "nominatim" + mode "755" + recursive true + end + + template "#{qa_bin_directory}/analyser/config/config.yaml" do + source "qa_config.erb" + owner "nominatim" + group "nominatim" + mode "755" + variables :outputdir => "#{qa_data_directory}/new" + end + + link "#{build_directory}/website/qa-data" do + to "#{qa_data_directory}/current" + owner "nominatim" + group "nominatim" + end end