+++ /dev/null
-#
-# Cookbook Name:: nominatim
-# Recipe:: base
-#
-# Copyright 2015, OpenStreetMap Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-include_recipe "apache::ssl"
-include_recipe "postgresql"
-
-package "php"
-package "php-cli"
-package "php-pgsql"
-package "php-fpm"
-package "php-pear"
-package "php-db"
-
-apache_module "rewrite"
-apache_module "proxy"
-apache_module "proxy_fcgi"
-apache_module "proxy_http"
-apache_module "headers"
-
-home_directory = data_bag_item("accounts", "nominatim")["home"]
-source_directory = "#{home_directory}/nominatim"
-email_errors = data_bag_item("accounts", "lonvia")["email"]
-
-database_cluster = node[:nominatim][:database][:cluster]
-database_version = database_cluster.sub(%r{/.*}, "")
-database_name = node[:nominatim][:database][:dbname]
-
-postgis_version = node[:nominatim][:database][:postgis]
-
-service "php7.0-fpm" do
- action [:enable, :start]
-end
-
-node[:nominatim][:fpm_pools].each do |name, data|
- template "/etc/php/7.0/fpm/pool.d/#{name}.conf" do
- source "fpm.conf.erb"
- owner "root"
- group "root"
- mode 0o644
- variables data.merge(:name => name, :port => data[:port])
- notifies :reload, "service[php7.0-fpm]"
- end
-end
-
-postgresql_munin "nominatim" do
- cluster database_cluster
- database database_name
-end
-
-directory node[:nominatim][:logdir] do
- owner "nominatim"
- group "nominatim"
- mode 0o755
- recursive true
-end
-
-file "#{node[:nominatim][:logdir]}/query.log" do
- action :create_if_missing
- owner "www-data"
- group "adm"
- mode 0o664
-end
-
-file "#{node[:nominatim][:logdir]}/update.log" do
- action :create_if_missing
- owner "nominatim"
- group "adm"
- mode 0o664
-end
-
-directory "#{home_directory}/status" do
- owner "nominatim"
- group "postgres"
- mode 0o775
-end
-
-template "/etc/logrotate.d/nominatim" do
- source "logrotate.nominatim.erb"
- owner "root"
- group "root"
- mode 0o644
-end
-
-template "/etc/logrotate.d/apache2" do
- source "logrotate.apache.erb"
- owner "root"
- group "root"
- mode 0o644
-end
-
-package "osmosis"
-package "gcc"
-package "proj-bin"
-package "libgeos-c1v5"
-package "postgresql-#{database_version}-postgis-#{postgis_version}"
-package "postgresql-server-dev-#{database_version}"
-package "build-essential"
-package "libxml2-dev"
-package "libexpat-dev"
-package "libgeos-dev"
-package "libgeos++-dev"
-package "libpq-dev"
-package "libbz2-dev"
-package "libtool"
-package "automake"
-package "libproj-dev"
-package "libprotobuf-c0-dev"
-package "protobuf-c-compiler"
-package "python-psycopg2"
-package "libboost-dev"
-package "libboost-system-dev"
-package "libboost-filesystem-dev"
-package "libboost-thread-dev"
-
-execute "compile_nominatim" do
- action :nothing
- command "cd #{source_directory} && ./autogen.sh && ./configure && make"
- user "nominatim"
-end
-
-template "#{source_directory}/.git/hooks/post-merge" do
- source "update_source.erb"
- owner "nominatim"
- group "nominatim"
- mode 0o755
- variables :source_directory => source_directory
-end
-
-template "#{source_directory}/settings/local.php" do
- source "nominatim.erb"
- owner "nominatim"
- group "nominatim"
- mode 0o664
- variables :postgres_version => database_version
-end
-
-template "#{source_directory}/settings/ip_blocks.conf" do
- action :create_if_missing
- source "ipblocks.erb"
- owner "nominatim"
- group "nominatim"
- mode 0o664
-end
-
-file "#{source_directory}/settings/apache_blocks.conf" do
- action :create_if_missing
- owner "nominatim"
- group "nominatim"
- mode 0o664
-end
-
-file "#{source_directory}/settings/ip_blocks.map" do
- action :create_if_missing
- owner "nominatim"
- group "nominatim"
- mode 0o664
-end
-
-cron_action = if node[:nominatim][:enabled]
- :create
- else
- :delete
- end
-
-template "/etc/cron.d/nominatim" do
- action cron_action
- source "cron.erb"
- owner "root"
- group "root"
- mode "0644"
- variables :bin_directory => "#{source_directory}/utils", :mailto => email_errors
-end
-
-template "#{source_directory}/utils/nominatim-update" do
- source "updater.erb"
- user "nominatim"
- group "nominatim"
- mode 0o755
- variables :bindir => source_directory,
- :srcdir => source_directory,
- :logfile => "#{node[:nominatim][:logdir]}/update.log",
- :branch => node[:nominatim][:revision],
- :update_stop_file => "#{home_directory}/status/updates_disabled"
-end
-
-template "/etc/init.d/nominatim-update" do
- source "updater.init.erb"
- user "nominatim"
- group "nominatim"
- mode 0o755
- variables :source_directory => source_directory
-end
-
-munin_plugin_conf "nominatim" do
- template "munin.erb"
- variables :db => node[:nominatim][:database][:dbname],
- :querylog => "#{node[:nominatim][:logdir]}/query.log"
-end
-
-munin_plugin "nominatim_importlag" do
- target "#{source_directory}/munin/nominatim_importlag"
-end
-
-munin_plugin "nominatim_query_speed" do
- target "#{source_directory}/munin/nominatim_query_speed_querylog"
-end
-
-munin_plugin "nominatim_requests" do
- target "#{source_directory}/munin/nominatim_requests_querylog"
-end
-
-munin_plugin "nominatim_throttled_ips" do
- target "#{source_directory}/munin/nominatim_throttled_ips"
-end
-
-external_data = [
- "wikipedia_article.sql.bin",
- "wikipedia_redirect.sql.bin",
- "gb_postcode_data.sql.gz"
-]
-
-external_data.each do |fname|
- remote_file "#{source_directory}/data/#{fname}" do
- action :create_if_missing
- source "http://www.nominatim.org/data/#{fname}"
- owner "nominatim"
- group "nominatim"
- mode 0o644
- end
-end
-
-additional_scripts = %w(backup-nominatim vacuum-db-nominatim)
-
-additional_scripts.each do |fname|
- template "/usr/local/bin/#{fname}" do
- source "#{fname}.erb"
- owner "root"
- group "root"
- mode 0o755
- variables :db => node[:nominatim][:database][:dbname]
- end
-end
-
-directory File.dirname(node[:nominatim][:flatnode_file]) do
- owner "nominatim"
- group "nominatim"
- mode 0o755
- recursive true
-end
-
-apache_site "nominatim.openstreetmap.org" do
- template "apache.erb"
- directory source_directory
- variables :pools => node[:nominatim][:fpm_pools]
-end
-
-apache_site "default" do
- action [:disable]
-end
-
-fail2ban_filter "nominatim" do
- failregex '^<HOST> - - \[[^]]+\] "[^"]+" 429 '
-end
-
-fail2ban_jail "nominatim" do
- filter "nominatim"
- logpath "/var/log/apache2/nominatim.openstreetmap.org-access.log"
- ports [80, 443]
- maxretry 100
-end
+++ /dev/null
-#
-# Cookbook Name:: nominatim
-# Recipe:: master
-#
-# Copyright 2015, OpenStreetMap Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-slaves = search(:node, "roles:nominatim-slave") # ~FC010
-
-node.default[:postgresql][:settings][:defaults][:late_authentication_rules] = []
-node.default[:rsyncd][:modules] = { :archive => { :hosts_allow => [] } }
-
-slaves.each do |slave|
- # set up DB access for each slave
- node.default[:postgresql][:settings][:defaults][:late_authentication_rules].push(
- :database => "replication",
- :user => "replication",
- :address => "#{slave[:networking][:internal_ipv4][:address]}/32"
- )
- # allow slaves access to the WAL logs
- node.default[:rsyncd][:modules][:archive][:hosts_allow].push(
- slave[:networking][:internal_ipv4][:address]
- )
-end
+++ /dev/null
-#
-# Cookbook Name:: nominatim
-# Recipe:: slave
-#
-# Copyright 2015, OpenStreetMap Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-master = search(:node, "roles:nominatim-master")[0] # ~FC010
-host = master[:nominatim][:master_host]
-
-node.default[:postgresql][:settings][:defaults][:primary_conninfo] = {
- :host => host,
- :port => "5432",
- :user => "replication",
- :passwords => { :bag => "nominatim", :item => "passwords" }
-}
-
-node.default[:postgresql][:settings][:defaults][:restore_command] =
- "/usr/bin/rsync #{host}::archive/%f %p"
+++ /dev/null
-#
-# Cookbook Name:: nominatim
-# Recipe:: standalone
-#
-# Copyright 2015, OpenStreetMap Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-include_recipe "git"
-
-database_cluster = node[:nominatim][:database][:cluster]
-home_directory = data_bag_item("accounts", "nominatim")["home"]
-
-git "#{home_directory}/nominatim" do
- action :checkout
- repository node[:nominatim][:repository]
- revision node[:nominatim][:revision]
- enable_submodules true
- user "nominatim"
- group "nominatim"
- notifies :run, "execute[compile_nominatim]"
-end
-
-include_recipe "nominatim::base"
-
-superusers = %w(tomh lonvia twain nominatim)
-
-superusers.each do |user|
- postgresql_user user do
- cluster database_cluster
- superuser true
- end
-end
-
-postgresql_user "www-data" do
- cluster database_cluster
-end
+++ /dev/null
-# Maintained by chef. DO NOT EDIT.
-
-MAILTO=<%= @mailto %>
-
-* * * * * lonvia <%= @bin_directory %>/cron_ipanalyse.py /var/log/apache2/nominatim.openstreetmap.org-access.log
-<% if node[:nominatim][:enable_backup] -%>
-00 3 1 * * nominatim /usr/local/bin/backup-nominatim
-<% end -%>
-30 1 * * * postgres /usr/local/bin/vacuum-db-nominatim
-<% if node[:postgresql][:settings][:defaults][:archive_mode] == "on" -%>
-05 */4 * * * postgres /usr/local/bin/clean-db-nominatim
-<% end -%>
+++ /dev/null
-<?php
-# DO NOT EDIT - This file is being maintained by Chef
-
-@define('CONST_Database_DSN', 'pgsql://@/<%= node[:nominatim][:database][:dbname] %>');
-if (isset($_SERVER['HTTPS']) && !empty($_SERVER['HTTPS']))
- @define('CONST_Website_BaseURL', 'https://nominatim.openstreetmap.org/');
-else
- @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/');
-
-@define('CONST_Postgresql_Version', '<%= @postgres_version %>');
-@define('CONST_Postgis_Version', '<%= node[:nominatim][:database][:postgis] %>');
-
-@define('CONST_Osm2pgsql_Flatnode_File', '<%= node[:nominatim][:flatnode_file] %>');
-@define('CONST_Search_NameOnlySearchFrequencyThreshold', 500);
-
-<% node[:nominatim][:tablespaces].each do |part,name| -%>
-@define('CONST_Tablespace_<%= part %>', '<%= name %>');
-<% end -%>
-
-@define('CONST_Log_File', '<%= node[:nominatim][:logdir] %>/query.log');
-@define('CONST_Log_DB', false);
+++ /dev/null
-#!/bin/bash
-
-# DO NOT EDIT - This file is being maintained by Chef
-
-cd <%= @source_directory %>
-
-git submodule update &&
-./autogen.sh &&
-./configure &&
-make
-
-psql -d <%= node[:nominatim][:database][:dbname] %> -c "SELECT version();" >/dev/null 2>&1
-if [[ "$?" == "0" ]]; then
- ./utils/setup.php --create-functions --create-partition-functions --enable-diff-updates
-fi
+++ /dev/null
-name "nominatim-master"
-description "Role applied to the master nominatim server"
-
-default_attributes(
- :postgresql => {
- :settings => {
- :defaults => {
- :wal_level => "hot_standby",
- :archive_mode => "on",
- :archive_command => "/bin/cp %p /data/postgresql-archive/%f",
- :max_wal_senders => "5"
- }
- }
- },
- :nominatim => {
- :state => "master",
- :enable_backup => true
- },
- :rsyncd => {
- :modules => {
- :archive => {
- :comment => "WAL Archive",
- :read_only => true,
- :write_only => false,
- :list => false,
- :uid => "postgres",
- :gid => "postgres",
- :transfer_logging => false
- }
- }
- }
-)
-
-run_list(
- "recipe[rsyncd]",
- "recipe[nominatim::master]",
- "role[nominatim-base]"
-)
+++ /dev/null
-name "nominatim-slave"
-description "Role applied to all slave nominatim servers"
-
-default_attributes(
- :postgresql => {
- :settings => {
- :defaults => {
- :hot_standby => "on",
- :hot_standby_feedback => "on",
- :standby_mode => "on"
- }
- }
- },
- :nominatim => {
- :state => "slave",
- :enable_backup => false
- }
-)
-
-run_list(
- "recipe[nominatim::slave]",
- "role[nominatim-base]"
-)
+++ /dev/null
-name "nominatim-standalone"
-description "Role applied to all stand-alone nominatim servers"
-
-default_attributes(
- :apt => {
- :sources => ["postgresql"]
- },
- :nominatim => {
- :enable_backup => false
- }
-)
-
-run_list(
- "role[nominatim]",
- "recipe[nominatim::standalone]"
-)
+++ /dev/null
-name "nominatim"
-description "Role applied to all nominatim servers"
-
-default_attributes(
- :accounts => {
- :users => {
- :lonvia => { :status => :administrator },
- :twain => { :status => :administrator },
- :nominatim => {
- :status => :role,
- :members => [:lonvia, :tomh, :twain]
- }
- }
- },
- :apache => {
- :mpm => "event",
- :timeout => 60,
- :keepalive => false,
- :event => {
- :server_limit => 32,
- :max_clients => 1600,
- :threads_per_child => 50
- }
- },
- :postgresql => {
- :settings => {
- :defaults => {
- :max_connections => "450",
- :synchronous_commit => "off",
- :checkpoint_segments => "32",
- :checkpoint_timeout => "10min",
- :max_wal_size => "1536MB",
- :checkpoint_completion_target => "0.9",
- :shared_buffers => "2GB",
- :autovacuum_max_workers => "1"
- }
- }
- },
- :sysctl => {
- :postgres => {
- :comment => "Increase shared memory for postgres",
- :parameters => {
- "kernel.shmmax" => 26 * 1024 * 1024 * 1024,
- "kernel.shmall" => 26 * 1024 * 1024 * 1024 / 4096
- }
- },
- :kernel_scheduler_tune => {
- :comment => "Tune kernel scheduler preempt",
- :parameters => {
- "kernel.sched_min_granularity_ns" => 10000000,
- "kernel.sched_wakeup_granularity_ns" => 15000000
- }
- },
- :swappiness => {
- :comment => "Reduce swap usage",
- :parameters => {
- "vm.swappiness" => 10
- }
- },
- :network_conntrack_time_wait => {
- :comment => "Only track completed connections for 30 seconds",
- :parameters => {
- "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30"
- }
- },
- :network_conntrack_max => {
- :comment => "Increase max number of connections tracked",
- :parameters => {
- "net.netfilter.nf_conntrack_max" => "131072"
- }
- }
- },
- :nominatim => {
- :enabled => false,
- :repository => "git://git.openstreetmap.org/nominatim.git",
- :tablespaces => {
- "Osm2pgsql_Data" => "data",
- "Osm2pgsql_Index" => "data",
- "Place_Data" => "data",
- "Place_Index" => "ssd2",
- "Address_Data" => "data",
- "Address_Index" => "ssd2",
- "Search_Data" => "ssd1",
- "Search_Index" => "ssd1",
- "Aux_Data" => "aux",
- "Aux_Index" => "aux"
- },
- :fpm_pools => {
- :www => {
- :port => "8000",
- :pm => "dynamic",
- :max_children => "60"
- },
- :bulk => {
- :port => "8001",
- :pm => "static",
- :max_children => "10"
- },
- :details => {
- :port => "8002",
- :pm => "static",
- :max_children => "2"
- }
- },
- :redirects => {}
- }
-)
:address => "2001:630:12:500:2e0:81ff:fec5:333e"
}
}
- },
- :postgresql => {
- :versions => ["9.3"],
- :settings => {
- :defaults => {
- :work_mem => "160MB",
- :maintenance_work_mem => "10GB",
- :random_page_cost => "1.5",
- :effective_cache_size => "40GB",
- :fsync => "on"
- }
- }
- },
- :nominatim => {
- :flatnode_file => "/ssd-old/nominatim/nodes.store",
- :database => {
- :cluster => "9.3/main",
- :dbname => "nominatim",
- :postgis => "2.1"
- },
- :fpm_pools => {
- :www => {
- :port => "8000",
- :pm => "dynamic",
- :max_children => "60"
- },
- :bulk => {
- :port => "8001",
- :pm => "static",
- :max_children => "10"
- }
- }
}
)
run_list(
"role[ic]",
- "role[tyan-s7010]",
- "role[nominatim-standalone]"
+ "role[tyan-s7010]"
)
projects / chef.git / commitdiff