~FC001
+~FC003
~FC064
~FC065
This cookbook installs and configures the Nominatim geocoding service.
-It contains four recipes:
-* base: common functionality for other recipes
-* standalone: run nominatim on a standalone server
-* master: run a master node in a replicated setup
-* slave: run a slave node in replicated setup
+It contains three recipes:
+* default: sets up a complete Nominatim installation including postgres backend
+ and apache frontend
+* master: defines additional attributes for running the server as DB master
+* slave: defines additional attributes for running the server as DB slave
+default[:nominatim][:state] = "off" # or: standalone, master, slave
+default[:nominatim][:dbadmins] = []
+default[:nominatim][:dbname] = "nominatim"
+default[:nominatim][:tablespaces] = []
default[:nominatim][:logdir] = "/var/log/nominatim"
+default[:nominatim][:repository] = "git://git.openstreetmap.org/nominatim.git"
default[:nominatim][:revision] = "master"
+default[:nominatim][:enable_backup] = false
+
+default[:nominatim][:fpm_pools] = {
+ :www => {
+ :port => "8000",
+ :pm => "dynamic",
+ :max_children => "60"
+ },
+ :bulk => {
+ :port => "8001",
+ :pm => "static",
+ :max_children => "10"
+ },
+ :details => {
+ :port => "8002",
+ :pm => "static",
+ :max_children => "2"
+ }
+}
+
+default[:nominatim][:redirects] = {}
--- /dev/null
+#
+# Cookbook Name:: nominatim
+# Recipe:: base
+#
+# Copyright 2015, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+basedir = data_bag_item("accounts", "nominatim")["home"]
+email_errors = data_bag_item("accounts", "lonvia")["email"]
+
+directory node[:nominatim][:logdir] do
+ owner "nominatim"
+ group "nominatim"
+ mode 0o755
+ recursive true
+end
+
+file "#{node[:nominatim][:logdir]}/query.log" do
+ action :create_if_missing
+ owner "www-data"
+ group "adm"
+ mode 0o664
+end
+
+file "#{node[:nominatim][:logdir]}/update.log" do
+ action :create_if_missing
+ owner "nominatim"
+ group "adm"
+ mode 0o664
+end
+
+directory "#{basedir}/status" do
+ owner "nominatim"
+ group "postgres"
+ mode 0o775
+end
+
+## Postgresql
+
+include_recipe "postgresql"
+
+package "postgis"
+
+node[:nominatim][:dbadmins].each do |user|
+ postgresql_user user do
+ cluster node[:nominatim][:dbcluster]
+ superuser true
+ only_if { node[:nominatim][:state] != "slave" }
+ end
+end
+
+postgresql_user "nominatim" do
+ cluster node[:nominatim][:dbcluster]
+ superuser true
+ only_if { node[:nominatim][:state] != "slave" }
+end
+
+postgresql_user "www-data" do
+ cluster node[:nominatim][:dbcluster]
+ only_if { node[:nominatim][:state] != "slave" }
+end
+
+postgresql_munin "nominatim" do
+ cluster node[:nominatim][:dbcluster]
+ database node[:nominatim][:dbname]
+end
+
+directory "#{basedir}/tablespaces" do
+ owner "postgres"
+ group "postgres"
+ mode 0o700
+end
+
+# Note: tablespaces must be exactly in the same location on each
+# Nominatim instance when replication is in use. Therefore
+# use symlinks to canonical directory locations.
+node[:nominatim][:tablespaces].each do |name, location|
+ directory location do
+ owner "postgres"
+ group "postgres"
+ mode 0o700
+ recursive true
+ end
+
+ link "#{basedir}/tablespaces/#{name}" do
+ to location
+ end
+
+ postgresql_tablespace name do
+ cluster node[:nominatim][:dbcluster]
+ location "#{basedir}/tablespaces/#{name}"
+ end
+end
+
+postgresql_user "replication" do
+ cluster node[:nominatim][:dbcluster]
+ password data_bag_item("nominatim", "passwords")["replication"]
+ replication true
+ only_if { node[:nominatim][:state] == "master" }
+end
+
+directory node[:rsyncd][:modules][:archive][:path] do
+ owner "postgres"
+ group "postgres"
+ mode 0o700
+ only_if { node[:nominatim][:state] == "master" }
+end
+
+template "/usr/local/bin/clean-db-nominatim" do
+ source "clean-db-nominatim.erb"
+ owner "root"
+ group "root"
+ mode 0o755
+ variables :archive_dir => node[:rsyncd][:modules][:archive][:path],
+ :update_stop_file => "#{basedir}/status/updates_disabled",
+ :streaming_clients => search(:node, "nominatim_state:slave").map { |slave| slave[:fdqn] }.join(" ")
+ only_if { node[:nominatim][:state] == "master" }
+end
+
+## Nominatim backend
+
+include_recipe "git"
+
+package "build-essential"
+package "cmake"
+package "g++"
+package "libboost-dev"
+package "libboost-system-dev"
+package "libboost-filesystem-dev"
+package "libexpat1-dev"
+package "zlib1g-dev"
+package "libxml2-dev"
+package "libbz2-dev"
+package "libpq-dev"
+package "libgeos++-dev"
+package "libproj-dev"
+package "osmosis"
+
+source_directory = "#{basedir}/nominatim"
+build_directory = "#{basedir}/bin"
+
+directory build_directory do
+ owner "nominatim"
+ group "nominatim"
+ mode 0o755
+ recursive true
+end
+
+# Normally syncing via chef is a bad idea because syncing might involve
+# an update of database functions which should not be done while an update
+# is ongoing. Therefore we sync in between update cycles. There is an
+# exception for slaves: they get DB function updates from the master, so
+# only the source code needs to be updated, which chef may do.
+git source_directory do
+ action node[:nominatim][:state] == "slave" ? :sync : :checkout
+ repository node[:nominatim][:repository]
+ revision node[:nominatim][:revision]
+ enable_submodules true
+ user "nominatim"
+ group "nominatim"
+ notifies :run, "execute[compile_nominatim]", :immediately
+end
+
+execute "compile_nominatim" do
+ action :nothing
+ user "nominatim"
+ cwd build_directory
+ command "cmake #{source_directory} && make"
+end
+
+template "#{source_directory}/.git/hooks/post-merge" do
+ source "git-post-merge-hook.erb"
+ owner "nominatim"
+ group "nominatim"
+ mode 0o755
+ variables :srcdir => source_directory,
+ :builddir => build_directory,
+ :dbname => node[:nominatim][:dbname]
+end
+
+template "#{build_directory}/settings/local.php" do
+ source "settings.erb"
+ owner "nominatim"
+ group "nominatim"
+ mode 0o664
+ variables :dbname => node[:nominatim][:dbname],
+ :flatnode_file => node[:nominatim][:flatnode_file],
+ :log_file => "#{node[:nominatim][:logdir]}/query.log"
+end
+
+if node[:nominatim][:flatnode_file] # ~FC023
+ directory File.dirname(node[:nominatim][:flatnode_file]) do
+ recursive true
+ end
+end
+
+template "/etc/logrotate.d/nominatim" do
+ source "logrotate.nominatim.erb"
+ owner "root"
+ group "root"
+ mode 0o644
+end
+
+external_data = [
+ "wikipedia_article.sql.bin",
+ "wikipedia_redirect.sql.bin",
+ "gb_postcode_data.sql.gz"
+]
+
+external_data.each do |fname|
+ remote_file "#{source_directory}/data/#{fname}" do
+ action :create_if_missing
+ source "http://www.nominatim.org/data/#{fname}"
+ owner "nominatim"
+ group "nominatim"
+ mode 0o644
+ end
+end
+
+template "/etc/cron.d/nominatim" do
+ action node[:nominatim][:state] == :off ? :delete : :create
+ source "nominatim.cron.erb"
+ owner "root"
+ group "root"
+ mode "0644"
+ variables :bin_directory => "#{source_directory}/utils", :mailto => email_errors
+end
+
+template "#{source_directory}/utils/nominatim-update" do
+ source "updater.erb"
+ user "nominatim"
+ group "nominatim"
+ mode 0o755
+end
+
+template "/etc/init.d/nominatim-update" do
+ source "updater.init.erb"
+ user "nominatim"
+ group "nominatim"
+ mode 0o755
+ variables :source_directory => source_directory
+end
+
+%w(backup-nominatim vacuum-db-nominatim).each do |fname|
+ template "/usr/local/bin/#{fname}" do
+ source "#{fname}.erb"
+ owner "root"
+ group "root"
+ mode 0o755
+ variables :db => node[:nominatim][:dbname]
+ end
+end
+
+## webserver frontend
+
+template "#{source_directory}/settings/ip_blocks.conf" do
+ action :create_if_missing
+ source "ipblocks.erb"
+ owner "nominatim"
+ group "nominatim"
+ mode 0o664
+end
+
+file "#{source_directory}/settings/apache_blocks.conf" do
+ action :create_if_missing
+ owner "nominatim"
+ group "nominatim"
+ mode 0o664
+end
+
+file "#{source_directory}/settings/ip_blocks.map" do
+ action :create_if_missing
+ owner "nominatim"
+ group "nominatim"
+ mode 0o664
+end
+
+include_recipe "apache::ssl"
+
+package "php"
+package "php-fpm"
+package "php-pgsql"
+package "php-pear"
+package "php-db"
+
+apache_module "rewrite"
+apache_module "proxy"
+apache_module "proxy_fcgi"
+apache_module "proxy_http"
+apache_module "headers"
+
+service "php5-fpm" do
+ if node[:lsb][:release].to_f >= 15.10
+ provider Chef::Provider::Service::Systemd
+ service_name "php7.0-fpm"
+ elsif node[:lsb][:release].to_f >= 14.04
+ provider Chef::Provider::Service::Upstart
+ end
+ action [:enable, :start]
+ supports :status => true, :restart => true, :reload => true
+end
+
+php_confdir = node[:lsb][:release].to_f >= 15.10 ? "/etc/php/7.0" : "/etc/php5"
+
+node[:nominatim][:fpm_pools].each do |name, data|
+ template "#{php_confdir}/fpm/pool.d/#{name}.conf" do
+ source "fpm.conf.erb"
+ owner "root"
+ group "root"
+ mode 0o644
+ variables data.merge(:name => name)
+ notifies :reload, "service[php5-fpm]"
+ end
+end
+
+apache_site "nominatim.openstreetmap.org" do
+ template "apache.erb"
+ directory source_directory
+ variables :pools => node[:nominatim][:fpm_pools]
+end
+
+apache_site "default" do
+ action [:disable]
+end
+
+template "/etc/logrotate.d/apache2" do
+ source "logrotate.apache.erb"
+ owner "root"
+ group "root"
+ mode 0o644
+end
+
+include_recipe "fail2ban"
+
+fail2ban_filter "nominatim" do
+ failregex '^<HOST> - - \[[^]]+\] "[^"]+" 429 '
+end
+
+fail2ban_jail "nominatim" do
+ filter "nominatim"
+ logpath "/var/log/apache2/nominatim.openstreetmap.org-access.log"
+ ports [80, 443]
+ maxretry 100
+end
+
+munin_plugin_conf "nominatim" do
+ template "munin.erb"
+ variables :db => node[:nominatim][:dbname],
+ :querylog => "#{node[:nominatim][:logdir]}/query.log"
+end
+
+munin_plugin "nominatim_importlag" do
+ target "#{source_directory}/munin/nominatim_importlag"
+end
+
+munin_plugin "nominatim_query_speed" do
+ target "#{source_directory}/munin/nominatim_query_speed_querylog"
+end
+
+munin_plugin "nominatim_requests" do
+ target "#{source_directory}/munin/nominatim_requests_querylog"
+end
+
+munin_plugin "nominatim_throttled_ips" do
+ target "#{source_directory}/munin/nominatim_throttled_ips"
+end
# limitations under the License.
#
-include_recipe "git"
-
-passwords = data_bag_item("nominatim", "passwords")
-database_cluster = node[:nominatim][:database][:cluster]
-home_directory = data_bag_item("accounts", "nominatim")["home"]
-
-wal_archives = node[:rsyncd][:modules][:archive][:path]
-# XXX we really should get a list of nominatim-slave nodes here
-slaves = "poldi"
-
-git "#{home_directory}/nominatim" do
- action :checkout
- repository node[:nominatim][:repository]
- enable_submodules true
- user "nominatim"
- group "nominatim"
- notifies :run, "execute[compile_nominatim]"
-end
-
-include_recipe "nominatim::base"
-
-superusers = %w(tomh lonvia twain nominatim)
-
-superusers.each do |user|
- postgresql_user user do
- cluster database_cluster
- superuser true
- end
-end
-
-postgresql_user "www-data" do
- cluster database_cluster
-end
-
-postgresql_user "replication" do
- cluster database_cluster
- password passwords["replication"]
- replication true
-end
-
-directory wal_archives do
- owner "postgres"
- group "postgres"
- mode 0o700
- only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
-end
-
-template "/usr/local/bin/clean-db-nominatim" do
- source "clean-db-nominatim.erb"
- owner "root"
- group "root"
- mode 0o755
- variables :archive_dir => wal_archives,
- :update_stop_file => "#{home_directory}/status/updates_disabled",
- :streaming_clients => slaves
- only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
+slaves = search(:node, "roles:nominatim-slave") # ~FC010
+
+node.default[:postgresql][:settings][:defaults][:late_authentication_rules] = []
+node.default[:rsyncd][:modules] = { :archive => { :hosts_allow => [] } }
+
+slaves.each do |slave|
+ # set up DB access for each slave
+ node.default[:postgresql][:settings][:defaults][:late_authentication_rules].push(
+ :database => "replication",
+ :user => "replication",
+ :address => "#{slave[:networking][:internal_ipv4][:address]}/32"
+ )
+ # allow slaves access to the WAL logs
+ node.default[:rsyncd][:modules][:archive][:hosts_allow].push(
+ slave[:networking][:internal_ipv4][:address]
+ )
end
# limitations under the License.
#
-include_recipe "git"
+master = search(:node, "roles:nominatim-master")[0] # ~FC010
+host = master[:nominatim][:master_host]
-home_directory = data_bag_item("accounts", "nominatim")["home"]
+node.default[:postgresql][:settings][:defaults][:primary_conninfo] = {
+ :host => host,
+ :port => "5432",
+ :user => "replication",
+ :passwords => { :bag => "nominatim", :item => "passwords" }
+}
-git "#{home_directory}/nominatim" do
- repository node[:nominatim][:repository]
- enable_submodules true
- user "nominatim"
- group "nominatim"
- notifies :run, "execute[compile_nominatim]"
-end
-
-include_recipe "nominatim::base"
+node.default[:postgresql][:settings][:defaults][:restore_command] =
+ "/usr/bin/rsync #{host}::archive/%f %p"
--- /dev/null
+#!/bin/bash
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+cd <%= @srcdir %>
+
+git submodule update
+
+cd <%= @builddir %>
+
+cmake <%= @srcdir %> &&
+make
+
+psql -d <%= @dbname %> -c "SELECT version();" >/dev/null 2>&1
+if [[ "$?" == "0" ]]; then
+ ./utils/setup.php --create-functions --create-partition-functions --enable-diff-updates
+fi
--- /dev/null
+# Maintained by chef. DO NOT EDIT.
+
+MAILTO=<%= @mailto %>
+
+* * * * * lonvia <%= @bin_directory %>/cron_ipanalyse.py /var/log/apache2/nominatim.openstreetmap.org-access.log
+<% if node[:nominatim][:enable_backup] -%>
+00 3 1 * * nominatim /usr/local/bin/backup-nominatim
+<% end -%>
+30 1 * * * postgres /usr/local/bin/vacuum-db-nominatim
+<% if node[:nominatim][:state] == "master" -%>
+05 */4 * * * postgres /usr/local/bin/clean-db-nominatim
+<% end -%>
--- /dev/null
+<?php
+# DO NOT EDIT - This file is being maintained by Chef
+
+@define('CONST_Database_DSN', 'pgsql://@/<%= @dbname %>');
+if (isset($_SERVER['HTTPS']) && !empty($_SERVER['HTTPS']))
+ @define('CONST_Website_BaseURL', 'https://nominatim.openstreetmap.org/');
+else
+ @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/');
+
+<% if @flatnode_file -%>
+@define('CONST_Osm2pgsql_Flatnode_File', '<%= @flatnode_file %>');
+<% end -%>
+@define('CONST_Search_NameOnlySearchFrequencyThreshold', 500);
+
+@define('CONST_Tablespace_Osm2pgsql_Data', 'dosm');
+@define('CONST_Tablespace_Osm2pgsql_Index', 'iosm');
+@define('CONST_Tablespace_Place_Data', 'dplace');
+@define('CONST_Tablespace_Place_Index', 'iplace');
+@define('CONST_Tablespace_Address_Data', 'daddress');
+@define('CONST_Tablespace_Address_Index', 'iaddress');
+@define('CONST_Tablespace_Search_Data', 'dsearch');
+@define('CONST_Tablespace_Search_Index', 'isearch');
+@define('CONST_Tablespace_Aux_Data', 'daux');
+@define('CONST_Tablespace_Aux_Index', 'iaux');
+
+@define('CONST_Log_File', '<%= @log_file %>');
+@define('CONST_Log_DB', false);
--- /dev/null
+name "nominatim-base"
+description "Role applied to all nominatim servers. Use for standalone server."
+
+default_attributes(
+ :accounts => {
+ :users => {
+ :lonvia => { :status => :administrator },
+ :twain => { :status => :administrator },
+ :nominatim => {
+ :status => :role,
+ :members => [:lonvia, :tomh, :twain]
+ }
+ }
+ },
+ :apache => {
+ :mpm => "event",
+ :timeout => 60,
+ :keepalive => false,
+ :event => {
+ :server_limit => 32,
+ :max_clients => 1600,
+ :threads_per_child => 50
+ }
+ },
+ :postgresql => {
+ :settings => {
+ :defaults => {
+ :max_connections => "450",
+ :synchronous_commit => "off",
+ :checkpoint_segments => "32",
+ :checkpoint_timeout => "10min",
+ :checkpoint_completion_target => "0.9",
+ :shared_buffers => "2GB",
+ :autovacuum_max_workers => "1"
+ }
+ }
+ },
+ :sysctl => {
+ :postgres => {
+ :comment => "Increase shared memory for postgres",
+ :parameters => {
+ "kernel.shmmax" => 26 * 1024 * 1024 * 1024,
+ "kernel.shmall" => 26 * 1024 * 1024 * 1024 / 4096
+ }
+ },
+ :kernel_scheduler_tune => {
+ :comment => "Tune kernel scheduler preempt",
+ :parameters => {
+ "kernel.sched_min_granularity_ns" => 10000000,
+ "kernel.sched_wakeup_granularity_ns" => 15000000
+ }
+ },
+ :swappiness => {
+ :comment => "Reduce swap usage",
+ :parameters => {
+ "vm.swappiness" => 10
+ }
+ },
+ :network_conntrack_time_wait => {
+ :comment => "Only track completed connections for 30 seconds",
+ :parameters => {
+ "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30"
+ }
+ },
+ :network_conntrack_max => {
+ :comment => "Increase max number of connections tracked",
+ :parameters => {
+ "net.netfilter.nf_conntrack_max" => "131072"
+ }
+ }
+ }
+)
+
+run_list(
+ "recipe[nominatim::default]"
+)
:wal_level => "hot_standby",
:archive_mode => "on",
:archive_command => "/bin/cp %p /data/postgresql-archive/%f",
- :max_wal_senders => "5",
- :late_authentication_rules => [
- { :database => "replication", :user => "replication", :address => "146.179.159.164/32" }
- ]
+ :max_wal_senders => "5"
}
}
},
:nominatim => {
- :enabled => true,
+ :state => "master",
:enable_backup => true
},
:rsyncd => {
:modules => {
:archive => {
:comment => "WAL Archive",
- :path => "/data/postgresql-archive",
:read_only => true,
:write_only => false,
:list => false,
:uid => "postgres",
:gid => "postgres",
- :transfer_logging => false,
- :hosts_allow => [
- "146.179.159.164"
- ]
+ :transfer_logging => false
}
}
}
)
run_list(
- "role[nominatim]",
+ "recipe[rsyncd]",
"recipe[nominatim::master]",
- "recipe[rsyncd]"
+ "role[nominatim-base]"
)
:defaults => {
:hot_standby => "on",
:hot_standby_feedback => "on",
- :standby_mode => "on",
- :primary_conninfo => {
- :host => "pummelzacken.ucl.openstreetmap.org",
- :port => "5432",
- :user => "replication",
- :passwords => { :bag => "nominatim", :item => "passwords" }
- },
- :restore_command => "/usr/bin/rsync pummelzacken.ucl.openstreetmap.org::archive/%f %p"
+ :standby_mode => "on"
}
}
},
:nominatim => {
+ :state => "slave",
:enable_backup => false
}
)
run_list(
- "role[nominatim]",
- "recipe[nominatim::slave]"
+ "recipe[nominatim::slave]",
+ "role[nominatim-base]"
)
projects / chef.git / commitdiff