From: Sarah Hoffmann Date: Fri, 19 Aug 2016 22:04:34 +0000 (+0200) Subject: reworke nominatim cookbook for cmake version of nominatim X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/4670fb28130f0c5f6371ddc4932bf856725ee589 reworke nominatim cookbook for cmake version of nominatim The cmake version can be found in the nominatim-base role, the previous standalone role remains in the transition period. --- diff --git a/cookbooks/nominatim/.foodcritic b/cookbooks/nominatim/.foodcritic index a085263c8..d8973b2fb 100644 --- a/cookbooks/nominatim/.foodcritic +++ b/cookbooks/nominatim/.foodcritic @@ -1,3 +1,4 @@ ~FC001 +~FC003 ~FC064 ~FC065 diff --git a/cookbooks/nominatim/README.md b/cookbooks/nominatim/README.md index 28c0ff364..190378ca2 100644 --- a/cookbooks/nominatim/README.md +++ b/cookbooks/nominatim/README.md @@ -2,8 +2,8 @@ This cookbook installs and configures the Nominatim geocoding service. -It contains four recipes: -* base: common functionality for other recipes -* standalone: run nominatim on a standalone server -* master: run a master node in a replicated setup -* slave: run a slave node in replicated setup +It contains three recipes: +* default: sets up a complete Nominatim installation including postgres backend + and apache frontend +* master: defines additional attributes for running the server as DB master +* slave: defines additional attributes for running the server as DB slave diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb index 8f187c777..ab273d84a 100644 --- a/cookbooks/nominatim/attributes/default.rb +++ b/cookbooks/nominatim/attributes/default.rb @@ -1,2 +1,28 @@ +default[:nominatim][:state] = "off" # or: standalone, master, slave +default[:nominatim][:dbadmins] = [] +default[:nominatim][:dbname] = "nominatim" +default[:nominatim][:tablespaces] = [] default[:nominatim][:logdir] = "/var/log/nominatim" +default[:nominatim][:repository] = "git://git.openstreetmap.org/nominatim.git" default[:nominatim][:revision] = "master" +default[:nominatim][:enable_backup] = false + +default[:nominatim][:fpm_pools] = { + :www => { + :port => "8000", + :pm => "dynamic", + :max_children => "60" + }, + :bulk => { + :port => "8001", + :pm => "static", + :max_children => "10" + }, + :details => { + :port => "8002", + :pm => "static", + :max_children => "2" + } +} + +default[:nominatim][:redirects] = {} diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb new file mode 100644 index 000000000..5b6eec45c --- /dev/null +++ b/cookbooks/nominatim/recipes/default.rb @@ -0,0 +1,378 @@ +# +# Cookbook Name:: nominatim +# Recipe:: base +# +# Copyright 2015, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +basedir = data_bag_item("accounts", "nominatim")["home"] +email_errors = data_bag_item("accounts", "lonvia")["email"] + +directory node[:nominatim][:logdir] do + owner "nominatim" + group "nominatim" + mode 0o755 + recursive true +end + +file "#{node[:nominatim][:logdir]}/query.log" do + action :create_if_missing + owner "www-data" + group "adm" + mode 0o664 +end + +file "#{node[:nominatim][:logdir]}/update.log" do + action :create_if_missing + owner "nominatim" + group "adm" + mode 0o664 +end + +directory "#{basedir}/status" do + owner "nominatim" + group "postgres" + mode 0o775 +end + +## Postgresql + +include_recipe "postgresql" + +package "postgis" + +node[:nominatim][:dbadmins].each do |user| + postgresql_user user do + cluster node[:nominatim][:dbcluster] + superuser true + only_if { node[:nominatim][:state] != "slave" } + end +end + +postgresql_user "nominatim" do + cluster node[:nominatim][:dbcluster] + superuser true + only_if { node[:nominatim][:state] != "slave" } +end + +postgresql_user "www-data" do + cluster node[:nominatim][:dbcluster] + only_if { node[:nominatim][:state] != "slave" } +end + +postgresql_munin "nominatim" do + cluster node[:nominatim][:dbcluster] + database node[:nominatim][:dbname] +end + +directory "#{basedir}/tablespaces" do + owner "postgres" + group "postgres" + mode 0o700 +end + +# Note: tablespaces must be exactly in the same location on each +# Nominatim instance when replication is in use. Therefore +# use symlinks to canonical directory locations. +node[:nominatim][:tablespaces].each do |name, location| + directory location do + owner "postgres" + group "postgres" + mode 0o700 + recursive true + end + + link "#{basedir}/tablespaces/#{name}" do + to location + end + + postgresql_tablespace name do + cluster node[:nominatim][:dbcluster] + location "#{basedir}/tablespaces/#{name}" + end +end + +postgresql_user "replication" do + cluster node[:nominatim][:dbcluster] + password data_bag_item("nominatim", "passwords")["replication"] + replication true + only_if { node[:nominatim][:state] == "master" } +end + +directory node[:rsyncd][:modules][:archive][:path] do + owner "postgres" + group "postgres" + mode 0o700 + only_if { node[:nominatim][:state] == "master" } +end + +template "/usr/local/bin/clean-db-nominatim" do + source "clean-db-nominatim.erb" + owner "root" + group "root" + mode 0o755 + variables :archive_dir => node[:rsyncd][:modules][:archive][:path], + :update_stop_file => "#{basedir}/status/updates_disabled", + :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fdqn] }.join(" ") + only_if { node[:nominatim][:state] == "master" } +end + +## Nominatim backend + +include_recipe "git" + +package "build-essential" +package "cmake" +package "g++" +package "libboost-dev" +package "libboost-system-dev" +package "libboost-filesystem-dev" +package "libexpat1-dev" +package "zlib1g-dev" +package "libxml2-dev" +package "libbz2-dev" +package "libpq-dev" +package "libgeos++-dev" +package "libproj-dev" +package "osmosis" + +source_directory = "#{basedir}/nominatim" +build_directory = "#{basedir}/bin" + +directory build_directory do + owner "nominatim" + group "nominatim" + mode 0o755 + recursive true +end + +# Normally syncing via chef is a bad idea because syncing might involve +# an update of database functions which should not be done while an update +# is ongoing. Therefore we sync in between update cycles. There is an +# exception for slaves: they get DB function updates from the master, so +# only the source code needs to be updated, which chef may do. +git source_directory do + action node[:nominatim][:state] == "slave" ? :sync : :checkout + repository node[:nominatim][:repository] + revision node[:nominatim][:revision] + enable_submodules true + user "nominatim" + group "nominatim" + notifies :run, "execute[compile_nominatim]", :immediately +end + +execute "compile_nominatim" do + action :nothing + user "nominatim" + cwd build_directory + command "cmake #{source_directory} && make" +end + +template "#{source_directory}/.git/hooks/post-merge" do + source "git-post-merge-hook.erb" + owner "nominatim" + group "nominatim" + mode 0o755 + variables :srcdir => source_directory, + :builddir => build_directory, + :dbname => node[:nominatim][:dbname] +end + +template "#{build_directory}/settings/local.php" do + source "settings.erb" + owner "nominatim" + group "nominatim" + mode 0o664 + variables :dbname => node[:nominatim][:dbname], + :flatnode_file => node[:nominatim][:flatnode_file], + :log_file => "#{node[:nominatim][:logdir]}/query.log" +end + +if node[:nominatim][:flatnode_file] # ~FC023 + directory File.dirname(node[:nominatim][:flatnode_file]) do + recursive true + end +end + +template "/etc/logrotate.d/nominatim" do + source "logrotate.nominatim.erb" + owner "root" + group "root" + mode 0o644 +end + +external_data = [ + "wikipedia_article.sql.bin", + "wikipedia_redirect.sql.bin", + "gb_postcode_data.sql.gz" +] + +external_data.each do |fname| + remote_file "#{source_directory}/data/#{fname}" do + action :create_if_missing + source "http://www.nominatim.org/data/#{fname}" + owner "nominatim" + group "nominatim" + mode 0o644 + end +end + +template "/etc/cron.d/nominatim" do + action node[:nominatim][:state] == :off ? :delete : :create + source "nominatim.cron.erb" + owner "root" + group "root" + mode "0644" + variables :bin_directory => "#{source_directory}/utils", :mailto => email_errors +end + +template "#{source_directory}/utils/nominatim-update" do + source "updater.erb" + user "nominatim" + group "nominatim" + mode 0o755 +end + +template "/etc/init.d/nominatim-update" do + source "updater.init.erb" + user "nominatim" + group "nominatim" + mode 0o755 + variables :source_directory => source_directory +end + +%w(backup-nominatim vacuum-db-nominatim).each do |fname| + template "/usr/local/bin/#{fname}" do + source "#{fname}.erb" + owner "root" + group "root" + mode 0o755 + variables :db => node[:nominatim][:dbname] + end +end + +## webserver frontend + +template "#{source_directory}/settings/ip_blocks.conf" do + action :create_if_missing + source "ipblocks.erb" + owner "nominatim" + group "nominatim" + mode 0o664 +end + +file "#{source_directory}/settings/apache_blocks.conf" do + action :create_if_missing + owner "nominatim" + group "nominatim" + mode 0o664 +end + +file "#{source_directory}/settings/ip_blocks.map" do + action :create_if_missing + owner "nominatim" + group "nominatim" + mode 0o664 +end + +include_recipe "apache::ssl" + +package "php" +package "php-fpm" +package "php-pgsql" +package "php-pear" +package "php-db" + +apache_module "rewrite" +apache_module "proxy" +apache_module "proxy_fcgi" +apache_module "proxy_http" +apache_module "headers" + +service "php5-fpm" do + if node[:lsb][:release].to_f >= 15.10 + provider Chef::Provider::Service::Systemd + service_name "php7.0-fpm" + elsif node[:lsb][:release].to_f >= 14.04 + provider Chef::Provider::Service::Upstart + end + action [:enable, :start] + supports :status => true, :restart => true, :reload => true +end + +php_confdir = node[:lsb][:release].to_f >= 15.10 ? "/etc/php/7.0" : "/etc/php5" + +node[:nominatim][:fpm_pools].each do |name, data| + template "#{php_confdir}/fpm/pool.d/#{name}.conf" do + source "fpm.conf.erb" + owner "root" + group "root" + mode 0o644 + variables data.merge(:name => name) + notifies :reload, "service[php5-fpm]" + end +end + +apache_site "nominatim.openstreetmap.org" do + template "apache.erb" + directory source_directory + variables :pools => node[:nominatim][:fpm_pools] +end + +apache_site "default" do + action [:disable] +end + +template "/etc/logrotate.d/apache2" do + source "logrotate.apache.erb" + owner "root" + group "root" + mode 0o644 +end + +include_recipe "fail2ban" + +fail2ban_filter "nominatim" do + failregex '^ - - \[[^]]+\] "[^"]+" 429 ' +end + +fail2ban_jail "nominatim" do + filter "nominatim" + logpath "/var/log/apache2/nominatim.openstreetmap.org-access.log" + ports [80, 443] + maxretry 100 +end + +munin_plugin_conf "nominatim" do + template "munin.erb" + variables :db => node[:nominatim][:dbname], + :querylog => "#{node[:nominatim][:logdir]}/query.log" +end + +munin_plugin "nominatim_importlag" do + target "#{source_directory}/munin/nominatim_importlag" +end + +munin_plugin "nominatim_query_speed" do + target "#{source_directory}/munin/nominatim_query_speed_querylog" +end + +munin_plugin "nominatim_requests" do + target "#{source_directory}/munin/nominatim_requests_querylog" +end + +munin_plugin "nominatim_throttled_ips" do + target "#{source_directory}/munin/nominatim_throttled_ips" +end diff --git a/cookbooks/nominatim/recipes/master.rb b/cookbooks/nominatim/recipes/master.rb index b135022b8..6dd15bf96 100644 --- a/cookbooks/nominatim/recipes/master.rb +++ b/cookbooks/nominatim/recipes/master.rb @@ -17,60 +17,20 @@ # limitations under the License. # -include_recipe "git" - -passwords = data_bag_item("nominatim", "passwords") -database_cluster = node[:nominatim][:database][:cluster] -home_directory = data_bag_item("accounts", "nominatim")["home"] - -wal_archives = node[:rsyncd][:modules][:archive][:path] -# XXX we really should get a list of nominatim-slave nodes here -slaves = "poldi" - -git "#{home_directory}/nominatim" do - action :checkout - repository node[:nominatim][:repository] - enable_submodules true - user "nominatim" - group "nominatim" - notifies :run, "execute[compile_nominatim]" -end - -include_recipe "nominatim::base" - -superusers = %w(tomh lonvia twain nominatim) - -superusers.each do |user| - postgresql_user user do - cluster database_cluster - superuser true - end -end - -postgresql_user "www-data" do - cluster database_cluster -end - -postgresql_user "replication" do - cluster database_cluster - password passwords["replication"] - replication true -end - -directory wal_archives do - owner "postgres" - group "postgres" - mode 0o700 - only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" } -end - -template "/usr/local/bin/clean-db-nominatim" do - source "clean-db-nominatim.erb" - owner "root" - group "root" - mode 0o755 - variables :archive_dir => wal_archives, - :update_stop_file => "#{home_directory}/status/updates_disabled", - :streaming_clients => slaves - only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" } +slaves = search(:node, "roles:nominatim-slave") # ~FC010 + +node.default[:postgresql][:settings][:defaults][:late_authentication_rules] = [] +node.default[:rsyncd][:modules] = { :archive => { :hosts_allow => [] } } + +slaves.each do |slave| + # set up DB access for each slave + node.default[:postgresql][:settings][:defaults][:late_authentication_rules].push( + :database => "replication", + :user => "replication", + :address => "#{slave[:networking][:internal_ipv4][:address]}/32" + ) + # allow slaves access to the WAL logs + node.default[:rsyncd][:modules][:archive][:hosts_allow].push( + slave[:networking][:internal_ipv4][:address] + ) end diff --git a/cookbooks/nominatim/recipes/slave.rb b/cookbooks/nominatim/recipes/slave.rb index 6c263caf5..22d1c0bf7 100644 --- a/cookbooks/nominatim/recipes/slave.rb +++ b/cookbooks/nominatim/recipes/slave.rb @@ -17,16 +17,15 @@ # limitations under the License. # -include_recipe "git" +master = search(:node, "roles:nominatim-master")[0] # ~FC010 +host = master[:nominatim][:master_host] -home_directory = data_bag_item("accounts", "nominatim")["home"] +node.default[:postgresql][:settings][:defaults][:primary_conninfo] = { + :host => host, + :port => "5432", + :user => "replication", + :passwords => { :bag => "nominatim", :item => "passwords" } +} -git "#{home_directory}/nominatim" do - repository node[:nominatim][:repository] - enable_submodules true - user "nominatim" - group "nominatim" - notifies :run, "execute[compile_nominatim]" -end - -include_recipe "nominatim::base" +node.default[:postgresql][:settings][:defaults][:restore_command] = + "/usr/bin/rsync #{host}::archive/%f %p" diff --git a/cookbooks/nominatim/templates/default/git-post-merge-hook.erb b/cookbooks/nominatim/templates/default/git-post-merge-hook.erb new file mode 100644 index 000000000..dbd3ddbdd --- /dev/null +++ b/cookbooks/nominatim/templates/default/git-post-merge-hook.erb @@ -0,0 +1,17 @@ +#!/bin/bash + +# DO NOT EDIT - This file is being maintained by Chef + +cd <%= @srcdir %> + +git submodule update + +cd <%= @builddir %> + +cmake <%= @srcdir %> && +make + +psql -d <%= @dbname %> -c "SELECT version();" >/dev/null 2>&1 +if [[ "$?" == "0" ]]; then + ./utils/setup.php --create-functions --create-partition-functions --enable-diff-updates +fi diff --git a/cookbooks/nominatim/templates/default/nominatim.cron.erb b/cookbooks/nominatim/templates/default/nominatim.cron.erb new file mode 100644 index 000000000..0cedfa360 --- /dev/null +++ b/cookbooks/nominatim/templates/default/nominatim.cron.erb @@ -0,0 +1,12 @@ +# Maintained by chef. DO NOT EDIT. + +MAILTO=<%= @mailto %> + +* * * * * lonvia <%= @bin_directory %>/cron_ipanalyse.py /var/log/apache2/nominatim.openstreetmap.org-access.log +<% if node[:nominatim][:enable_backup] -%> +00 3 1 * * nominatim /usr/local/bin/backup-nominatim +<% end -%> +30 1 * * * postgres /usr/local/bin/vacuum-db-nominatim +<% if node[:nominatim][:state] == "master" -%> +05 */4 * * * postgres /usr/local/bin/clean-db-nominatim +<% end -%> diff --git a/cookbooks/nominatim/templates/default/settings.erb b/cookbooks/nominatim/templates/default/settings.erb new file mode 100644 index 000000000..89c6febcc --- /dev/null +++ b/cookbooks/nominatim/templates/default/settings.erb @@ -0,0 +1,27 @@ +'); +if (isset($_SERVER['HTTPS']) && !empty($_SERVER['HTTPS'])) + @define('CONST_Website_BaseURL', 'https://nominatim.openstreetmap.org/'); +else + @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/'); + +<% if @flatnode_file -%> +@define('CONST_Osm2pgsql_Flatnode_File', '<%= @flatnode_file %>'); +<% end -%> +@define('CONST_Search_NameOnlySearchFrequencyThreshold', 500); + +@define('CONST_Tablespace_Osm2pgsql_Data', 'dosm'); +@define('CONST_Tablespace_Osm2pgsql_Index', 'iosm'); +@define('CONST_Tablespace_Place_Data', 'dplace'); +@define('CONST_Tablespace_Place_Index', 'iplace'); +@define('CONST_Tablespace_Address_Data', 'daddress'); +@define('CONST_Tablespace_Address_Index', 'iaddress'); +@define('CONST_Tablespace_Search_Data', 'dsearch'); +@define('CONST_Tablespace_Search_Index', 'isearch'); +@define('CONST_Tablespace_Aux_Data', 'daux'); +@define('CONST_Tablespace_Aux_Index', 'iaux'); + +@define('CONST_Log_File', '<%= @log_file %>'); +@define('CONST_Log_DB', false); diff --git a/roles/nominatim-base.rb b/roles/nominatim-base.rb new file mode 100644 index 000000000..502627fbc --- /dev/null +++ b/roles/nominatim-base.rb @@ -0,0 +1,76 @@ +name "nominatim-base" +description "Role applied to all nominatim servers. Use for standalone server." + +default_attributes( + :accounts => { + :users => { + :lonvia => { :status => :administrator }, + :twain => { :status => :administrator }, + :nominatim => { + :status => :role, + :members => [:lonvia, :tomh, :twain] + } + } + }, + :apache => { + :mpm => "event", + :timeout => 60, + :keepalive => false, + :event => { + :server_limit => 32, + :max_clients => 1600, + :threads_per_child => 50 + } + }, + :postgresql => { + :settings => { + :defaults => { + :max_connections => "450", + :synchronous_commit => "off", + :checkpoint_segments => "32", + :checkpoint_timeout => "10min", + :checkpoint_completion_target => "0.9", + :shared_buffers => "2GB", + :autovacuum_max_workers => "1" + } + } + }, + :sysctl => { + :postgres => { + :comment => "Increase shared memory for postgres", + :parameters => { + "kernel.shmmax" => 26 * 1024 * 1024 * 1024, + "kernel.shmall" => 26 * 1024 * 1024 * 1024 / 4096 + } + }, + :kernel_scheduler_tune => { + :comment => "Tune kernel scheduler preempt", + :parameters => { + "kernel.sched_min_granularity_ns" => 10000000, + "kernel.sched_wakeup_granularity_ns" => 15000000 + } + }, + :swappiness => { + :comment => "Reduce swap usage", + :parameters => { + "vm.swappiness" => 10 + } + }, + :network_conntrack_time_wait => { + :comment => "Only track completed connections for 30 seconds", + :parameters => { + "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30" + } + }, + :network_conntrack_max => { + :comment => "Increase max number of connections tracked", + :parameters => { + "net.netfilter.nf_conntrack_max" => "131072" + } + } + } +) + +run_list( + "recipe[nominatim::default]" +) diff --git a/roles/nominatim-master.rb b/roles/nominatim-master.rb index 85baaf892..1961d835c 100644 --- a/roles/nominatim-master.rb +++ b/roles/nominatim-master.rb @@ -8,38 +8,31 @@ default_attributes( :wal_level => "hot_standby", :archive_mode => "on", :archive_command => "/bin/cp %p /data/postgresql-archive/%f", - :max_wal_senders => "5", - :late_authentication_rules => [ - { :database => "replication", :user => "replication", :address => "146.179.159.164/32" } - ] + :max_wal_senders => "5" } } }, :nominatim => { - :enabled => true, + :state => "master", :enable_backup => true }, :rsyncd => { :modules => { :archive => { :comment => "WAL Archive", - :path => "/data/postgresql-archive", :read_only => true, :write_only => false, :list => false, :uid => "postgres", :gid => "postgres", - :transfer_logging => false, - :hosts_allow => [ - "146.179.159.164" - ] + :transfer_logging => false } } } ) run_list( - "role[nominatim]", + "recipe[rsyncd]", "recipe[nominatim::master]", - "recipe[rsyncd]" + "role[nominatim-base]" ) diff --git a/roles/nominatim-slave.rb b/roles/nominatim-slave.rb index a161d90cd..0ef9a9f51 100644 --- a/roles/nominatim-slave.rb +++ b/roles/nominatim-slave.rb @@ -7,23 +7,17 @@ default_attributes( :defaults => { :hot_standby => "on", :hot_standby_feedback => "on", - :standby_mode => "on", - :primary_conninfo => { - :host => "pummelzacken.ucl.openstreetmap.org", - :port => "5432", - :user => "replication", - :passwords => { :bag => "nominatim", :item => "passwords" } - }, - :restore_command => "/usr/bin/rsync pummelzacken.ucl.openstreetmap.org::archive/%f %p" + :standby_mode => "on" } } }, :nominatim => { + :state => "slave", :enable_backup => false } ) run_list( - "role[nominatim]", - "recipe[nominatim::slave]" + "recipe[nominatim::slave]", + "role[nominatim-base]" )