reintroduce cutoffs when searching for very frequent words

[nominatim.git] / vagrant / Install-on-Ubuntu-20.sh

diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh
old mode 100644 (file)
new mode 100755 (executable)
index 9b79add..57361ec
--- a/vagrant/Install-on-Ubuntu-20.sh
+++ b/vagrant/Install-on-Ubuntu-20.sh
@@ -1,17 +1,10 @@
-#!/bin/bash
+#!/bin/bash -e

 #
 # hacks for broken vagrant box      #DOCS:
 sudo rm -f /var/lib/dpkg/lock       #DOCS:
 #
 # hacks for broken vagrant box      #DOCS:
 sudo rm -f /var/lib/dpkg/lock       #DOCS:

-sudo update-locale LANG=en_US.UTF-8 #DOCS:

 export APT_LISTCHANGES_FRONTEND=none #DOCS:
 export DEBIAN_FRONTEND=noninteractive #DOCS:
 
 export APT_LISTCHANGES_FRONTEND=none #DOCS:
 export DEBIAN_FRONTEND=noninteractive #DOCS:
 

-#
-# !!! danger "Important"
-#     Ubuntu 20.04 uses Postgresql 12 and Postgis 3, which are known to cause
-#     performance issues. They are not recommended for a production
-#     installation at the moment.
-#

 # *Note:* these installation instructions are also available in executable
 #         form for use with vagrant under vagrant/Install-on-Ubuntu-20.sh.
 #
 # *Note:* these installation instructions are also available in executable
 #         form for use with vagrant under vagrant/Install-on-Ubuntu-20.sh.
 #


@@ -20,33 +13,28 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 #
 # These instructions expect that you have a freshly installed Ubuntu 20.04.
 #
 #
 # These instructions expect that you have a freshly installed Ubuntu 20.04.
 #

-# Make sure all packages are are up-to-date by running:
+# Make sure all packages are up-to-date by running:

 #
 
 #
 

-#DOCS:    :::sh
-    sudo apt-get \ #DOCS:
-        -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" \ #DOCS:
-        --allow-downgrades --allow-remove-essential --allow-change-held-packages \ #DOCS:
-        -fuy install grub-pc #DOCS:

     sudo apt-get update -qq
 
 # Now you can install all packages needed for Nominatim:
 
     sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
     sudo apt-get update -qq
 
 # Now you can install all packages needed for Nominatim:
 
     sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \

-                            libboost-filesystem-dev libexpat1-dev zlib1g-dev \
-                            libbz2-dev libpq-dev libproj-dev \
-                            postgresql-server-dev-12 postgresql-12-postgis-3 \
-                            postgresql-contrib postgresql-12-postgis-3-scripts \
-                            apache2 php php-pgsql libapache2-mod-php \
-                            php-intl python3-setuptools python3-dev python3-pip \
-                            python3-psycopg2 python3-tidylib git
+                        libboost-filesystem-dev libexpat1-dev zlib1g-dev \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \
+                        nlohmann-json3-dev postgresql-12-postgis-3 \
+                        postgresql-contrib-12 postgresql-12-postgis-3-scripts \
+                        libicu-dev python3-dotenv \
+                        python3-psycopg2 python3-psutil python3-jinja2 python3-pip \
+                        python3-icu python3-datrie python3-yaml git
+
+# Some of the Python packages that come with Ubuntu 20.04 are too old, so
+# install the latest version from pip:

 
 

-# If you want to run the test suite, you need to install the following
-# additional packages:
+    pip3 install --user sqlalchemy asyncpg

 
 

-    sudo apt-get install -y phpunit php-codesniffer php-cgi

 
 

-    pip3 install --user behave nose

 #
 # System Configuration
 # ====================
 #
 # System Configuration
 # ====================


@@ -67,11 +55,18 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 #
 # You may find a more suitable location if you wish.
 #
 #
 # You may find a more suitable location if you wish.
 #

+# The following instructions assume you are logged in as this user.
+# You can also switch to the user with:
+#
+#     sudo -u nominatim bash
+#

 # To be able to copy and paste instructions from this manual, export
 # user name and home directory now like this:
 #
 # To be able to copy and paste instructions from this manual, export
 # user name and home directory now like this:
 #

+if [ "x$USERNAME" == "x" ]; then #DOCS:

     export USERNAME=vagrant        #DOCS:    export USERNAME=nominatim
     export USERHOME=/home/vagrant  #DOCS:    export USERHOME=/srv/nominatim
     export USERNAME=vagrant        #DOCS:    export USERNAME=nominatim
     export USERHOME=/home/vagrant  #DOCS:    export USERHOME=/srv/nominatim

+fi                                 #DOCS:

 #
 # **Never, ever run the installation as a root user.** You have been warned.
 #
 #
 # **Never, ever run the installation as a root user.** You have been warned.
 #


@@ -83,14 +78,17 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 # ---------------------
 #
 # Tune the postgresql configuration, which is located in 
 # ---------------------
 #
 # Tune the postgresql configuration, which is located in 

-# `/etc/postgresql/12/main/postgresql.conf`. See section *Postgres Tuning* in
-# [the installation page](../admin/Installation.md#postgresql-tuning)
+# `/etc/postgresql/12/main/postgresql.conf`. See section *Tuning the PostgreSQL database*
+# in [the installation page](../admin/Installation.md#tuning-the-postgresql-database)

 # for the parameters to change.
 #
 # Restart the postgresql service after updating this config file.
 
 # for the parameters to change.
 #
 # Restart the postgresql service after updating this config file.
 

+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo pg_ctlcluster 12 main start  #DOCS:
+else                                  #DOCS:

     sudo systemctl restart postgresql
     sudo systemctl restart postgresql

-
+fi                                    #DOCS:

 #
 # Finally, we need to add two postgres users: one for the user that does
 # the import and another for the webserver which should access the database
 #
 # Finally, we need to add two postgres users: one for the user that does
 # the import and another for the webserver which should access the database


@@ -100,35 +98,6 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo -u postgres createuser -s $USERNAME
     sudo -u postgres createuser www-data
 
     sudo -u postgres createuser -s $USERNAME
     sudo -u postgres createuser www-data
 

-#
-# Setting up the Apache Webserver
-# -------------------------------
-#
-# You need to create an alias to the website directory in your apache
-# configuration. Add a separate nominatim configuration to your webserver:
-
-#DOCS:```sh
-sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
-<Directory "$USERHOME/build/website"> #DOCS:<Directory "$USERHOME/Nominatim/build/website">
-  Options FollowSymLinks MultiViews
-  AddType text/html   .php
-  DirectoryIndex search.php
-  Require all granted
-</Directory>
-
-Alias /nominatim $USERHOME/build/website  #DOCS:Alias /nominatim $USERHOME/Nominatim/build/website
-EOFAPACHECONF
-#DOCS:```
-
-sudo sed -i 's:#.*::' /etc/apache2/conf-available/nominatim.conf #DOCS:
-
-#
-# Then enable the configuration and restart apache
-#
-
-    sudo a2enconf nominatim
-    sudo systemctl restart apache2
-

 #
 # Installing Nominatim
 # ====================
 #
 # Installing Nominatim
 # ====================


@@ -140,7 +109,7 @@ sudo sed -i 's:#.*::' /etc/apache2/conf-available/nominatim.conf #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME

-    git clone --recursive git://github.com/openstreetmap/Nominatim.git
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git

     cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
     cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:


@@ -150,28 +119,182 @@ fi                                 #DOCS:
 # download the country grid:
 
 if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
 # download the country grid:
 
 if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh

-    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz

 fi                                 #DOCS:
 
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
 fi                                 #DOCS:
 
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 

-    cd $USERHOME                   #DOCS:    :::sh
-    mkdir build
-    cd build
+    mkdir $USERHOME/build
+    cd $USERHOME/build

     cmake $USERHOME/Nominatim
     make
     cmake $USERHOME/Nominatim
     make

+    sudo make install
+
+# Nominatim is now ready to use. You can continue with
+# [importing a database from OSM data](../admin/Import.md). If you want to set up
+# the API frontend first, continue reading.
+#
+# Setting up the Python frontend
+# ==============================
+#
+# Some of the Python packages in Ubuntu are too old. Therefore run the
+# frontend from a Python virtualenv with current packages.
+#
+# To set up the virtualenv, run:
+
+#DOCS:```sh
+sudo apt-get install -y virtualenv
+virtualenv $USERHOME/nominatim-venv
+$USERHOME/nominatim-venv/bin/pip install SQLAlchemy PyICU psycopg[binary] \
+              psycopg2-binary python-dotenv PyYAML falcon uvicorn gunicorn
+#DOCS:```
+
+# Next you need to create a systemd job that runs Nominatim on gunicorn.
+# First create a systemd job that manages the socket file:
+
+#DOCS:```sh
+sudo tee /etc/systemd/system/nominatim.socket << EOFSOCKETSYSTEMD
+[Unit]
+Description=Gunicorn socket for Nominatim
+
+[Socket]
+ListenStream=/run/nominatim.sock
+SocketUser=www-data
+
+[Install]
+WantedBy=multi-user.target
+EOFSOCKETSYSTEMD
+#DOCS:```
+
+# Then create the service for Nominatim itself.
+
+#DOCS:```sh
+sudo tee /etc/systemd/system/nominatim.service << EOFNOMINATIMSYSTEMD
+[Unit]
+Description=Nominatim running as a gunicorn application
+After=network.target
+Requires=nominatim.socket
+
+[Service]
+Type=simple
+Environment="PYTHONPATH=/usr/local/lib/nominatim/lib-python/"
+User=www-data
+Group=www-data
+WorkingDirectory=$USERHOME/nominatim-project
+ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim.server.falcon.server:run_wsgi
+ExecReload=/bin/kill -s HUP \$MAINPID
+StandardOutput=append:/var/log/gunicorn-nominatim.log
+StandardError=inherit
+PrivateTmp=true
+TimeoutStopSec=5
+KillMode=mixed
+
+[Install]
+WantedBy=multi-user.target
+EOFNOMINATIMSYSTEMD
+#DOCS:```
+
+# Activate the services:
+
+if [ "x$NOSYSTEMD" != "xyes" ]; then  #DOCS:
+    sudo systemctl daemon-reload
+    sudo systemctl enable nominatim.socket
+    sudo systemctl start nominatim.socket
+    sudo systemctl enable nominatim.service
+fi                                    #DOCS:

 
 

-# You need to create a minimal configuration file that tells nominatim
-# where it is located on the webserver:
+
+# Setting up a webserver
+# ======================
+#
+# The webserver is only needed as a proxy between the public interface
+# and the gunicorn service.
+#
+# The frontend will need configuration information from the project
+# directory, which will be populated later
+# [during the import process](../admin/Import.md#creating-the-project-directory)
+# Already create the project directory itself now:
+
+
+    mkdir $USERHOME/nominatim-project
+
+
+# Option 1: Using Apache
+# ----------------------
+#
+if [ "x$2" == "xinstall-apache" ]; then #DOCS:
+# First install apache itself and enable the proxy module:
+
+    sudo apt-get install -y apache2
+    sudo a2enmod proxy_http
+
+# To set up proxying for Apache add the following configuration:

 
 #DOCS:```sh
 
 #DOCS:```sh

-tee settings/local.php << EOF
-<?php
- @define('CONST_Website_BaseURL', '/nominatim/');
-EOF
+sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
+ProxyPass /nominatim "unix:/run/nominatim.sock|http://localhost/"
+EOFAPACHECONF

 #DOCS:```
 
 #DOCS:```
 

+#
+# Then enable the configuration and restart apache
+#
+
+    sudo a2enconf nominatim
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo apache2ctl start             #DOCS:
+else                                  #DOCS:
+    sudo systemctl restart apache2
+fi                                    #DOCS:
+
+# The Nominatim API is now available at `http://localhost/nominatim/`.
+
+fi   #DOCS:
+
+#
+# Option 2: Using nginx
+# ---------------------
+#
+if [ "x$2" == "xinstall-nginx" ]; then #DOCS:
+
+# First install nginx itself:
+
+    sudo apt-get install -y nginx
+
+# Then create a Nginx configuration to forward http requests to that socket.
+
+#DOCS:```sh
+sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    root $USERHOME/nominatim-project/website;
+    index /search;
+
+    location /nominatim/ {
+            proxy_set_header Host \$http_host;
+            proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto \$scheme;
+            proxy_redirect off;
+            proxy_pass http://unix:/run/nominatim.sock:/;
+    }
+}
+EOF_NGINX_CONF
+#DOCS:```
+
+# Enable the configuration and restart Nginx
+#
+
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo /usr/sbin/nginx &            #DOCS:
+else                                  #DOCS:
+    sudo systemctl restart nginx
+fi                                    #DOCS:
+
+# The Nominatim API is now available at `http://localhost/`.
+
+

 
 

-# Nominatim is now ready to use. Continue with
-# [importing a database from OSM data](../admin/Import-and-Update.md).
+fi   #DOCS: