From 7aa1c692da44b5441554522d486e01aa02b27c3a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 5 Jul 2013 14:01:47 +0200 Subject: [PATCH] bring nominatim under chef control --- cookbooks/nominatim/metadata.rb | 1 + cookbooks/nominatim/recipes/default.rb | 175 ++++++++++++++++-- .../nominatim/templates/default/apache.erb | 13 +- .../nominatim/templates/default/ipblocks.erb | 23 +++ .../nominatim/templates/default/nominatim.erb | 6 + .../templates/default/update_source.erb | 14 ++ .../nominatim/templates/default/updater.erb | 30 +++ .../templates/default/updater.init.erb | 25 +++ roles/nominatim.rb | 23 ++- 9 files changed, 285 insertions(+), 25 deletions(-) create mode 100644 cookbooks/nominatim/templates/default/ipblocks.erb create mode 100644 cookbooks/nominatim/templates/default/nominatim.erb create mode 100644 cookbooks/nominatim/templates/default/update_source.erb create mode 100644 cookbooks/nominatim/templates/default/updater.erb create mode 100644 cookbooks/nominatim/templates/default/updater.init.erb diff --git a/cookbooks/nominatim/metadata.rb b/cookbooks/nominatim/metadata.rb index 2806429bc..2f8e20ee4 100644 --- a/cookbooks/nominatim/metadata.rb +++ b/cookbooks/nominatim/metadata.rb @@ -6,3 +6,4 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc')) version "1.0.0" depends "apache" depends "postgresql" +depends "git" diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 0d653c638..a33fe85c2 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -19,16 +19,22 @@ include_recipe "apache" include_recipe "postgresql" +include_recipe "git" package "php5" package "php5-cli" package "php5-pgsql" - +package "php5-fpm" +package "php-pear" package "php-apc" apache_module "rewrite" apache_module "fastcgi-handler" +home_directory = data_bag_item("accounts", "nominatim")["home"] +source_directory = "#{home_directory}/nominatim" +email_errors = data_bag_item("accounts", "lonvia")["email"] + service "php5-fpm" do action [ :enable, :start ] supports :status => true, :restart => true, :reload => true @@ -37,24 +43,19 @@ end apache_site "nominatim.openstreetmap.org" do template "apache.erb" directory "/home/lonvia/nominatim" + variables :pools => node[:nominatim][:fpm_pools] end -template "/etc/php5/fpm/pool.d/www.conf" do - source "fpm.conf.erb" - owner "root" - group "root" - mode 0644 - variables :name => "www", :pm => "dynamic", :max_children => "50" - notifies :reload, resources(:service => "php5-fpm") -end +node[:nominatim][:fpm_pools].each do |name,data| -template "/etc/php5/fpm/pool.d/bulk.conf" do - source "fpm.conf.erb" - owner "root" - group "root" - mode 0644 - variables :name => "bulk", :pm => "static", :max_children => "7" - notifies :reload, resources(:service => "php5-fpm") + template "/etc/php5/fpm/pool.d/#{name}.conf" do + source "fpm.conf.erb" + owner "root" + group "root" + mode 0644 + variables data.merge(:name => name) + notifies :reload, resources(:service => "php5-fpm") + end end postgresql_user "tomh" do @@ -72,6 +73,11 @@ postgresql_user "twain" do superuser true end +postgresql_user "nominatim" do + cluster "9.1/main" + superuser true +end + postgresql_user "www-data" do cluster "9.1/main" end @@ -80,3 +86,140 @@ postgresql_munin "nominatim" do cluster "9.1/main" database "nominatim" end + +directory "/var/log/nominatim" do + owner "nominatim" + group "nominatim" + mode 0755 +end + +package "osmosis" +package "gcc" +package "proj-bin" +package "libgeos-c1" +package "postgresql-9.1-postgis" +package "postgresql-server-dev-9.1" +package "build-essential" +package "libxml2-dev" +package "libgeos-dev" +package "libgeos++-dev" +package "libpq-dev" +package "libbz2-dev" +package "libtool" +package "automake" +package "libproj-dev" +package "libprotobuf-c0-dev" +package "protobuf-c-compiler" + +execute "php-pear-db" do + command "pear install DB" + not_if { File.exists?("/usr/share/php/DB") } +end + +execute "compile_nominatim" do + action :nothing + command "cd #{source_directory} && ./autogen.sh && ./configure && make" + user "nominatim" +end + +git source_directory do + action :checkout + repository node[:nominatim][:repository] + enable_submodules true + user "nominatim" + group "nominatim" + notifies :run, "execute[compile_nominatim]" +end + +directory "#{source_directory}/log" do + owner "nominatim" + group "nominatim" + mode 0755 +end + + +template "#{source_directory}/.git/hooks/post-merge" do + source "update_source.erb" + owner "nominatim" + group "nominatim" + mode 0755 + variables :source_directory => source_directory +end + +template "#{source_directory}/settings/local.php" do + source "nominatim.erb" + owner "nominatim" + group "nominatim" + mode 0664 +end + +template "#{source_directory}/settings/ip_blocks.conf" do + action :create_if_missing + source "ipblocks.erb" + owner "nominatim" + group "nominatim" + mode 0664 +end + +file "#{source_directory}/settings/apache_blocks.conf" do + action :create_if_missing + owner "nominatim" + group "nominatim" + mode 0664 +end + +file "#{source_directory}/settings/ip_blocks.map" do + action :create_if_missing + owner "nominatim" + group "nominatim" + mode 0664 +end + +cron "nominatim_logrotate" do + hour "5" + minute "30" + weekday "0" + command "#{source_directory}/utils/cron_logrotate.sh" + user "nominatim" + mailto email_errors +end + +cron "nominatim_banip" do + command "#{source_directory}/utils/cron_banip.sh" + user "nominatim" + mailto email_errors +end + +cron "nominatim_vacuum" do + hour "2" + minute "00" + command "#{source_directory}/utils/cron_vacuum.sh" + user "nominatim" + mailto email_errors +end + +['search', 'reverse'].each do |filename| + ['phpj', 'phpx'].each do |ext| + link "#{source_directory}/website/#{filename}.#{ext}" do + to "#{source_directory}/website/#{filename}.php" + user "nominatim" + group "nominatim" + end + end +end + +template "#{source_directory}/utils/nominatim-update" do + source "updater.erb" + user "nominatim" + group "nominatim" + mode 0755 +end + +template "/etc/init.d/nominatim-update" do + source "updater.init.erb" + user "nominatim" + group "nominatim" + mode 0755 + variables :source_directory => source_directory +end + diff --git a/cookbooks/nominatim/templates/default/apache.erb b/cookbooks/nominatim/templates/default/apache.erb index de4fb525e..add708d69 100644 --- a/cookbooks/nominatim/templates/default/apache.erb +++ b/cookbooks/nominatim/templates/default/apache.erb @@ -24,18 +24,15 @@ AddHandler fcgi:/var/run/php5-fpm-www.sock .phpj - Alias /pool-www/ "<%= @directory %>/website/" - + <% @pools.each do |name,details| -%> + Alias /pool-<%= name %>/ "<%= @directory %>/website/" + > AddHandler fcgi:/var/run/php5-fpm-www.sock .php AddHandler fcgi:/var/run/php5-fpm-www.sock .phpx AddHandler fcgi:/var/run/php5-fpm-www.sock .phpj - Alias /pool-bulk/ "<%= @directory %>/website/" - - AddHandler fcgi:/var/run/php5-fpm-bulk.sock .php - AddHandler fcgi:/var/run/php5-fpm-bulk.sock .phpx - AddHandler fcgi:/var/run/php5-fpm-bulk.sock .phpj - + <% end -%> + Redirect 509 /pool-block/ ErrorDocument 509 /509.html diff --git a/cookbooks/nominatim/templates/default/ipblocks.erb b/cookbooks/nominatim/templates/default/ipblocks.erb new file mode 100644 index 000000000..5d0f87f85 --- /dev/null +++ b/cookbooks/nominatim/templates/default/ipblocks.erb @@ -0,0 +1,23 @@ +LOGFILE=/var/log/nominatim/restricted_ips.log +# space-separated list of IPs that are never banned +WHITELIST = '' +# space-separated list of IPs manually blocked +BLACKLIST = '' +# user-agents that should be blocked from bulk mode +# (matched with startswith) +UA_BLOCKLIST = () + +# time before a automatically blocked IP is allowed back +BLOCKCOOLOFF_PERIOD='1 hour' +# quiet time before an IP is released from the bulk pool +BULKCOOLOFF_PERIOD='15 min' + +BULKLONG_LIMIT=8000 +BULKSHORT_LIMIT=2000 +BLOCK_UPPER=19000 +BLOCK_LOWER=4000 +BLOCK_LOADFAC=380 +BULK_LOADFAC=160 +BULK_LOWER=1500 +MAX_BULK_IPS=85 + diff --git a/cookbooks/nominatim/templates/default/nominatim.erb b/cookbooks/nominatim/templates/default/nominatim.erb new file mode 100644 index 000000000..ea486ad24 --- /dev/null +++ b/cookbooks/nominatim/templates/default/nominatim.erb @@ -0,0 +1,6 @@ +'); +@define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/'); + diff --git a/cookbooks/nominatim/templates/default/update_source.erb b/cookbooks/nominatim/templates/default/update_source.erb new file mode 100644 index 000000000..49e285df9 --- /dev/null +++ b/cookbooks/nominatim/templates/default/update_source.erb @@ -0,0 +1,14 @@ +#!/bin/bash + +# DO NOT EDIT - This file is being maintained by Chef + +cd <%= @source_directory %> + +./autogen.sh && +./configure && +make + +psql -U postgres <%= node[:nominatim][:database][:dbname] %> -c "SELECT version();" >/dev/null 2>&1 +if [ ! $? ]; then + ./utils/setup.php --create-functions --create-partition-functions --enable-diff-updates +fi diff --git a/cookbooks/nominatim/templates/default/updater.erb b/cookbooks/nominatim/templates/default/updater.erb new file mode 100644 index 000000000..2c553ad21 --- /dev/null +++ b/cookbooks/nominatim/templates/default/updater.erb @@ -0,0 +1,30 @@ +#!/bin/bash -e + +# DO NOT EDIT - This file is being maintained by Chef + +# Send output to the log +exec > /var/log/nominatim/update.log 2>&1 + +# Change to Nominatim directory +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd $DIR/.. + +while true +do + starttime=`date +%s` + + ./utils/update.php --no-npi --import-osmosis + + # will trigger recompilation if necessary + git pull + + # sleep a bit if updates take less than a minute + endtime=`date +%s` + elapsed=$((endtime - starttime)) + if [[ $elapsed < 60 ]] + then + sleepy=$((60 - $elapsed)) + echo "Sleeping for ${sleepy}s..." + sleep $sleepy + fi +done diff --git a/cookbooks/nominatim/templates/default/updater.init.erb b/cookbooks/nominatim/templates/default/updater.init.erb new file mode 100644 index 000000000..0afbf97b1 --- /dev/null +++ b/cookbooks/nominatim/templates/default/updater.init.erb @@ -0,0 +1,25 @@ +#!/bin/bash + +# DO NOT EDIT - This file is being maintained by Chef + +start() { + start-stop-daemon --start --chuid suzuki --background --make-pidfile --pidfile /var/run/nominatim-update.pid --exec <%= @source_directory %>/utils/nominatim-update +} + +stop() { + start-stop-daemon --stop --retry 3600 --pidfile /var/run/nominatim-update.pid +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop || exit $? + start + ;; +esac + diff --git a/roles/nominatim.rb b/roles/nominatim.rb index 401f1f755..ccb9cf0d5 100644 --- a/roles/nominatim.rb +++ b/roles/nominatim.rb @@ -5,7 +5,11 @@ default_attributes( :accounts => { :users => { :lonvia => { :status => :administrator }, - :twain => { :status => :administrator } + :twain => { :status => :administrator }, + :nominatim => { + :status => :role, + :members => [ :lonvia, :tomh, :twain ] + }, } }, :apache => { @@ -41,6 +45,23 @@ default_attributes( "kernel.shmall" => 16 * 1024 * 1024 * 1024 / 4096 } } + }, + :nominatim => { + :repository => "git://git.openstreetmap.org/nominatim.git", + :database => { + :cluster => "9.1/main", + :dbname => "nominatim" + }, + :fpm_pools => { + :www => { + :pm => "dynamic", + :max_children => "50" + }, + :bulk => { + :pm => "static", + :max_children => "7" + } + } } ) -- 2.39.5