add vagrant script for ubuntu 16 and polish everything

diff --git a/Vagrantfile b/Vagrantfile
index 09e7eab9361271c1e623099d46b5d21b8d497745..96d03a2e0ea5021cf18a69e3112077bddc19edf6 100644 (file)
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -8,21 +8,28 @@ Vagrant.configure("2") do |config|
   # If true, then any SSH connections made will enable agent forwarding.
   config.ssh.forward_agent = true
 
   # If true, then any SSH connections made will enable agent forwarding.
   config.ssh.forward_agent = true
 

+  checkout = "yes"

   if ENV['CHECKOUT'] != 'y' then
       config.vm.synced_folder ".", "/home/vagrant/Nominatim"
   if ENV['CHECKOUT'] != 'y' then
       config.vm.synced_folder ".", "/home/vagrant/Nominatim"

+      checkout = "no"

   end
 
   config.vm.define "ubuntu" do |sub|
   end
 
   config.vm.define "ubuntu" do |sub|

-      sub.vm.box = "ubuntu/trusty64"
-      sub.vm.provision :shell, :path => "vagrant/ubuntu-trusty-provision.sh"
-  end
-  config.vm.define "ubuntu-php7" do |sub|
-      sub.vm.box = "ubuntu/trusty64"
-      sub.vm.provision :shell, :path => "vagrant/ubuntu-trusty-php7-provision.sh"
+      sub.vm.box = "bento/ubuntu-16.04"
+      sub.vm.provision :shell do |s|
+        s.path = "vagrant/install-on-ubuntu-16.sh"
+        s.privileged = false
+        s.args = [checkout]
+      end

   end
   end

+

   config.vm.define "centos" do |sub|
       sub.vm.box = "bento/centos-7.2"
   config.vm.define "centos" do |sub|
       sub.vm.box = "bento/centos-7.2"

-      sub.vm.provision :shell, :path => "vagrant/install-on-centos-7.sh"
+      sub.vm.provision :shell do |s|
+        s.path = "vagrant/install-on-centos-7.sh"
+        s.privileged = false
+        s.args = [checkout]
+      end

   end
 
   # configure shared package cache if possible
   end
 
   # configure shared package cache if possible

diff --git a/docs/Installation.md b/docs/Installation.md
new file mode 100644 (file)
index 0000000..ddd0fc7
--- /dev/null
+++ b/docs/Installation.md
@@ -0,0 +1,151 @@
+Nominatim Installation
+======================
+
+This page contains generic installation instructions for Nominatim and its
+prerequisites. There are also step-by-step instructions available for
+the following operating systems:
+
+  * [Ubuntu 16.04](install-on-ubuntu-16.md)
+  * [CentOS 7.2](install-on-centos-7.md)
+
+These OS-specific instructions can also be found in executable form
+in the `vagrant/` directory.
+
+Prerequisites
+-------------
+
+### Software
+
+For compiling:
+
+  * [cmake](https://cmake.org/)
+  * [libxml2](http://xmlsoft.org/)
+  * a recent C++ compiler
+
+Nominatim comes with its own version of osm2pgsql. See the
+[osm2pgsql README](../osm2pgsql/README.md) for additional dependencies
+required for compiling osm2pgsql.
+
+For running tests:
+
+  * [lettuce](http://lettuce.it)
+  * [Shapely](http://toblerity.org/shapely/index.html)
+  * [Psycopg2](http://initd.org/psycopg)
+  * [nose](https://nose.readthedocs.io)
+  * [phpunit](https://phpunit.de)
+
+For running Nominatim:
+
+  * [PostgreSQL](http://www.postgresql.org) (9.1 or later)
+  * [PostGIS](http://postgis.refractions.net) (2.0 or later)
+  * [PHP](http://php.net)
+  * PHP-pgsql
+  * [PEAR::DB](http://pear.php.net/package/DB)
+  * a webserver (apache or nginx are recommended)
+
+For running continuous updates:
+
+  * [osmosis](http://wiki.openstreetmap.org/wiki/Osmosis)
+
+### Hardware
+
+A minimum of 2GB of RAM is required or installation will fail. For a full
+planet import 32GB of RAM or more strongly are recommended.
+
+For a full planet install you will need about 500GB of hard disk space (as of
+June 2016, take into account that the OSM database is growing fast). SSD disks
+will help considerably to speed up import and queries.
+
+On a 6-core machine with 32GB RAM and SSDs the import of a full planet takes
+a bit more than 2 days. Without SSDs 7-8 days are more realistic.
+
+
+Setup of the Server
+-------------------
+
+### PostgreSQL Tuning
+
+You might want to tune your PostgreSQL installation so that the later steps
+make best use of your hardware. You should tune the following parameters in
+your `postgresql.conf` file.
+
+    shared_buffers (2GB)
+    maintenance_work_mem (10GB)
+    work_mem (50MB)
+    effective_cache_size (24GB)
+    synchronous_commit = off
+    checkpoint_segments = 100
+    checkpoint_timeout = 10min
+    checkpoint_completion_target = 0.9
+
+The numbers in brackets behind some parameters seem to work fine for
+32GB RAM machine. Adjust to your setup.
+
+For the initial import, you should also set:
+
+    fsync = off
+    full_page_writes = off
+
+Don't forget to reenable them after the initial import or you risk database
+corruption. Autovacuum must not be switched off because it ensures that the
+tables are frequently analysed.
+
+### Webserver setup
+
+The `website/` directory in the build directory contains the configured
+website. Include the directory into your webbrowser to serve php files
+from there.
+
+#### Configure for use with Apache
+
+Make sure your Apache configuration contains the required permissions for the
+directory and create an alias:
+
+    <Directory "/srv/nominatim/build/website">
+      Options FollowSymLinks MultiViews
+      AddTpe text/html   .php
+      Require all granted
+    </Directory>
+    Alias /nominatim /srv/nominatim/build/website
+
+`/srv/nominatim/build` should be replaced with the location of your
+build directory.
+
+After making changes in the apache config you need to restart apache.
+The website should now be available on http://localhost/nominatim.
+
+#### Configure for use with Nginx
+
+Use php-fpm as a deamon for serving PHP cgi. Install php-fpm together with nginx.
+
+By default php listens on a network socket. If you want it to listen to a
+Unix socket instead, change the pool configuration (`pool.d/www.conf`) as
+follows:
+
+    ; Comment out the tcp listener and add the unix socket
+    ;listen = 127.0.0.1:9000
+    listen = /var/run/php5-fpm.sock
+
+    ; Ensure that the daemon runs as the correct user
+    listen.owner = www-data
+    listen.group = www-data
+    listen.mode = 0666
+
+Tell nginx that php files are special and to fastcgi_pass to the php-fpm
+unix socket by adding the location definition to the default configuration.
+
+    location ~ [^/]\.php(/|$) {
+        fastcgi_split_path_info ^(.+?\.php)(/.*)$;
+        if (!-f $document_root$fastcgi_script_name) {
+            return 404;
+        }
+        fastcgi_pass unix:/var/run/php5-fpm.sock;
+        fastcgi_index index.php;
+        include fastcgi.conf;
+    }
+
+Restart the nginx and php5-fpm services and the website should now be available
+on http://localhost/.
+
+
+Now continue with [importing the database](Import_and_update.md).

diff --git a/vagrant/install-on-centos-7.sh b/vagrant/install-on-centos-7.sh
index afdeccd94f794c1f2b5a7febf56998ede3b1cb7f..d76a816fbfb652dad64a713d869602ea2dc9cad2 100755 (executable)
--- a/vagrant/install-on-centos-7.sh
+++ b/vagrant/install-on-centos-7.sh
@@ -24,6 +24,14 @@
                         php-pgsql php php-pear php-pear-DB libpqxx-devel proj-epsg \
                         bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel
 
                         php-pgsql php php-pear php-pear-DB libpqxx-devel proj-epsg \
                         bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel
 

+# If you want to run the test suite, you need to install the following
+# aditional packages:
+
+    sudo yum install -y python-pip python-Levenshtein python-psycopg2 \
+                        php-phpunit-PHPUnit
+    pip install --user --upgrade pip setuptools lettuce==0.2.18 six==1.9 \
+                                 haversine Shapely pytidylib
+

 #
 # System Configuration
 # ====================
 #
 # System Configuration
 # ====================


@@ -90,7 +98,7 @@
 # configuration. Add a separate nominatim configuration to your webserver:
 
 #DOCS:```
 # configuration. Add a separate nominatim configuration to your webserver:
 
 #DOCS:```

-sudo cat > /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF
+sudo tee /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF

 <Directory "$USERHOME/build/website"> #DOCS:<Directory "$USERHOME/Nominatim/build/website">
   Options FollowSymLinks MultiViews
   AddType text/html   .php
 <Directory "$USERHOME/build/website"> #DOCS:<Directory "$USERHOME/Nominatim/build/website">
   Options FollowSymLinks MultiViews
   AddType text/html   .php


@@ -130,10 +138,10 @@ sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS:
 #
 # Get the source code from Github and change into the source directory
 #
 #
 # Get the source code from Github and change into the source directory
 #

-if [ "x$CHECKOUT" == "xy" ]; then  #DOCS:
+if [ "x$1" == "xyes" ]; then  #DOCS:

 
     cd $USERHOME
 
     cd $USERHOME

-    sudo -u $USERNAME git clone --recursive git://github.com/twain47/Nominatim.git
+    git clone --recursive git://github.com/twain47/Nominatim.git

 #DOCS:    cd Nominatim
 
 else                               #DOCS:
 #DOCS:    cd Nominatim
 
 else                               #DOCS:


@@ -143,21 +151,22 @@ fi                                 #DOCS:
 # The code is built in a special directory. Create this directory,
 # then configure and build Nominatim in there:
 
 # The code is built in a special directory. Create this directory,
 # then configure and build Nominatim in there:
 

-    sudo -u $USERNAME mkdir build
+    mkdir build

     cd build
     cd build

-    sudo -u $USERNAME cmake $USERHOME/Nominatim
-    sudo -u $USERNAME make
+    cmake $USERHOME/Nominatim
+    make

 
 # You need to create a minimal configuration file that tells nominatim
 
 # You need to create a minimal configuration file that tells nominatim

-# the name of your webserver user:
+# the name of your webserver user and the URL of the website:

 
 #DOCS:```
 
 #DOCS:```

-sudo -u $USERNAME cat > settings/local.php << EOF
+tee settings/local.php << EOF

 <?php
  @define('CONST_Database_Web_User', 'apache');
 <?php
  @define('CONST_Database_Web_User', 'apache');

+ @define('CONST_Website_BaseURL', '/nominatim/');

 EOF
 #DOCS:```
 
 
 EOF
 #DOCS:```
 
 

-Nominatim is now ready to use. Continue with
-[importing a database from OSM data](Import_and_update.md).
+# Nominatim is now ready to use. Continue with
+# [importing a database from OSM data](Import_and_update.md).

diff --git a/vagrant/install-on-ubuntu-16.sh b/vagrant/install-on-ubuntu-16.sh
new file mode 100755 (executable)
index 0000000..f7af07c
--- /dev/null
+++ b/vagrant/install-on-ubuntu-16.sh
@@ -0,0 +1,157 @@
+#!/bin/bash
+#
+# hacks for broken vagrant box      #DOCS:
+sudo rm -f /var/lib/dpkg/lock       #DOCS:
+sudo update-locale LANG=en_US.UTF-8 #DOCS:
+#
+# *Note:* these installation instructions are also available in executable
+#         form for use with vagrant under vagrant/install-on-ubuntu-16.sh.
+#
+# Installing the Required Software
+# ================================
+#
+# These instructions expect that you have a freshly installed Ubuntu 16.04.
+#
+# Make sure all packages are are up-to-date by running:
+#
+
+    sudo apt-get update -qq
+    sudo apt-get upgrade -y
+
+# Now you can install all packages needed for Nominatim:
+
+    sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
+                            libboost-filesystem-dev libexpat1-dev zlib1g-dev libxml2-dev\
+                            libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev \
+                            postgresql-server-dev-9.5 postgresql-9.5-postgis-2.2 postgresql-contrib-9.5 \
+                            apache2 php php-pgsql libapache2-mod-php php-pear php-db \
+                            git
+
+# If you want to run the test suite, you need to install the following
+# aditional packages:
+
+    sudo apt-get install -y python-dev python-pip python-levenshtein python-shapely \
+                            python-psycopg2 tidy python-nose python-tidylib \
+                            phpunit
+
+    pip install --user lettuce==0.2.18 six==1.7 haversine
+
+#
+# System Configuration
+# ====================
+#
+# The following steps are meant to configure a fresh Ubuntu installation
+# for use with Nominatim. You may skip some of the steps if you have your
+# OS already configured.
+#
+# Creating Dedicated User Accounts
+# --------------------------------
+#
+# Nominatim will run as a global service on your machine. It is therefore
+# best to install it under its own separate user account. In the following
+# we assume this user is called nominatim and the installation will be in
+# /srv/nominatim. To create the user and directory run:
+#
+#     sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
+#
+# You may find a more suitable location if you wish.
+#
+# To be able to copy and paste instructions from this manual, export
+# user name and home directory now like this:
+#
+    export USERNAME=vagrant        #DOCS:    export USERNAME=nominatim
+    export USERHOME=/home/vagrant  #DOCS:    export USERHOME=/srv/nominatim
+#
+# **Never, ever run the installation as a root user.** You have been warned.
+#
+# Make sure that system servers can read from the home directory:
+
+    chmod a+x $USERHOME
+
+# Setting up PostgreSQL
+# ---------------------
+#
+# Tune the postgresql configuration, which is located in 
+# `/etc/postgresql/9.5/main/postgresql.conf`. See section *Postgres Tuning* in
+# [the installation page](Installation.md) for the parameters to change.
+#
+# Restart the postgresql service after updating this config file.
+
+    sudo systemctl restart postgresql
+
+#
+# Finally, we need to add two postgres users: one for the user that does
+# the import and another for the webserver ro access the database:
+#
+
+    sudo -u postgres createuser -s $USERNAME
+    sudo -u postgres createuser www-data
+
+#
+# Setting up the Apache Webserver
+# -------------------------------
+#
+# You need to create an alias to the website directory in your apache
+# configuration. Add a separate nominatim configuration to your webserver:
+
+#DOCS:```
+sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
+<Directory "$USERHOME/build/website"> #DOCS:<Directory "$USERHOME/Nominatim/build/website">
+  Options FollowSymLinks MultiViews
+  AddType text/html   .php
+  Require all granted
+</Directory>
+
+Alias /nominatim $USERHOME/build/website  #DOCS:Alias /nominatim $USERHOME/Nominatim/build/website
+EOFAPACHECONF
+#DOCS:```
+
+sudo sed -i 's:#.*::' /etc/apache2/conf-available/nominatim.conf #DOCS:
+
+#
+# Then enable the configuration and restart apache
+#
+
+    sudo a2enconf nominatim
+    sudo systemctl restart apache2
+
+#
+# Installing Nominatim
+# ====================
+#
+# Building and Configuration
+# --------------------------
+#
+# Get the source code from Github and change into the source directory
+#
+if [ "x$1" == "xyes" ]; then  #DOCS:
+
+    cd $USERHOME
+    git clone --recursive git://github.com/twain47/Nominatim.git
+#DOCS:    cd Nominatim
+
+else                               #DOCS:
+    cd $USERHOME                   #DOCS:
+fi                                 #DOCS:
+
+# The code is built in a special directory. Create this directory,
+# then configure and build Nominatim in there:
+
+    mkdir build
+    cd build
+    cmake $USERHOME/Nominatim
+    make
+
+# You need to create a minimal configuration file that tells nominatim
+# where it is located on the webserver:
+
+#DOCS:```
+tee settings/local.php << EOF
+<?php
+ @define('CONST_Website_BaseURL', '/nominatim/');
+EOF
+#DOCS:```
+
+
+# Nominatim is now ready to use. Continue with
+# [importing a database from OSM data](Import_and_update.md).