From e2416f90af990126e0f624ccb076cebd49485aec Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 24 Jun 2024 20:07:02 +0200 Subject: [PATCH 1/1] installation for Ubuntu 24.04 with packages Also remove Ubuntu 20.04 instructions. --- Makefile | 6 +- Vagrantfile | 18 ++--- docs/CMakeLists.txt | 2 +- docs/admin/Installation.md | 60 +++++++++----- docs/mkdocs.yml | 2 +- packaging/nominatim-db/pyproject.toml | 2 +- vagrant/Install-on-Ubuntu-22.sh | 2 +- ...n-Ubuntu-20.sh => Install-on-Ubuntu-24.sh} | 81 +++++++++---------- 8 files changed, 97 insertions(+), 76 deletions(-) rename vagrant/{Install-on-Ubuntu-20.sh => Install-on-Ubuntu-24.sh} (80%) mode change 100755 => 100644 diff --git a/Makefile b/Makefile index 1575b72a..bea0e6c9 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,13 @@ all: build: build-core build-db build-api build-core: - cd packaging/nominatim-core; python3 -m build . --outdir ../dist/ + python3 -m build packaging/nominatim-core --outdir dist/ build-db: - cd packaging/nominatim-db; python3 -m build . --outdir ../dist/ + python3 -m build packaging/nominatim-db --outdir dist/ build-api: - cd packaging/nominatim-api; python3 -m build . --outdir ../dist/ + python3 -m build packaging/nominatim-api --outdir dist/ # Tests diff --git a/Vagrantfile b/Vagrantfile index 7f5f2459..753b1833 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -69,28 +69,28 @@ Vagrant.configure("2") do |config| end end - config.vm.define "ubuntu20" do |sub| - sub.vm.box = "generic/ubuntu2004" + config.vm.define "ubuntu24" do |sub| + sub.vm.box = "bento/ubuntu-24.04" sub.vm.provision :shell do |s| - s.path = "vagrant/Install-on-Ubuntu-20.sh" + s.path = "vagrant/Install-on-Ubuntu-24.sh" s.privileged = false s.args = [checkout] end end - config.vm.define "ubuntu20-apache" do |sub| - sub.vm.box = "generic/ubuntu2004" + config.vm.define "ubuntu24-apache" do |sub| + sub.vm.box = "bento/ubuntu-24.04" sub.vm.provision :shell do |s| - s.path = "vagrant/Install-on-Ubuntu-20.sh" + s.path = "vagrant/Install-on-Ubuntu-24.sh" s.privileged = false s.args = [checkout, "install-apache"] end end - config.vm.define "ubuntu20-nginx" do |sub| - sub.vm.box = "generic/ubuntu2004" + config.vm.define "ubuntu24-nginx" do |sub| + sub.vm.box = "bento/ubuntu-24.04" sub.vm.provision :shell do |s| - s.path = "vagrant/Install-on-Ubuntu-20.sh" + s.path = "vagrant/Install-on-Ubuntu-24.sh" s.privileged = false s.args = [checkout, "install-nginx"] end diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 637ecfe9..8f864ec3 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -24,7 +24,7 @@ foreach (src ${DOC_SOURCES}) endforeach() ADD_CUSTOM_TARGET(doc - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-24.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-24.md COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md COMMAND mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml ) diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index ef6bd081..d6cdf751 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -4,8 +4,8 @@ This page contains generic installation instructions for Nominatim and its prerequisites. There are also step-by-step instructions available for the following operating systems: + * [Ubuntu 24.04](../appendix/Install-on-Ubuntu-24.md) * [Ubuntu 22.04](../appendix/Install-on-Ubuntu-22.md) - * [Ubuntu 20.04](../appendix/Install-on-Ubuntu-20.md) These OS-specific instructions can also be found in executable form in the `vagrant/` directory. @@ -27,34 +27,42 @@ and can't offer support. otherwise import and queries will be slow to the point of being unusable. Query performance has marked improvements with PostgreSQL 13+ and PostGIS 3.2+. -For compiling: - - * [cmake](https://cmake.org/) - * [expat](https://libexpat.github.io/) - * [proj](https://proj.org/) - * [bzip2](http://www.bzip.org/) - * [zlib](https://www.zlib.net/) - * [ICU](http://site.icu-project.org/) - * [nlohmann/json](https://json.nlohmann.me/) - * [Boost libraries](https://www.boost.org/), including system and filesystem - * PostgreSQL client libraries - * a recent C++ compiler (gcc 5+ or Clang 3.8+) - For running Nominatim: * [PostgreSQL](https://www.postgresql.org) (9.6+ will work, 11+ strongly recommended) * [PostGIS](https://postgis.net) (2.2+ will work, 3.0+ strongly recommended) + * [osm2pgsql](https://osm2pgsql.org) (1.8+, optional when building with CMake) * [Python 3](https://www.python.org/) (3.7+) + +Furthermore the following Python libraries are required: + * [Psycopg2](https://www.psycopg.org) (2.7+) * [Python Dotenv](https://github.com/theskumar/python-dotenv) * [psutil](https://github.com/giampaolo/psutil) * [Jinja2](https://palletsprojects.com/p/jinja/) * [SQLAlchemy](https://www.sqlalchemy.org/) (1.4.31+ with greenlet support) - * [asyncpg](https://magicstack.github.io/asyncpg) (0.8+) + * one of + * [psycopg3](https://www.psycopg.org) + * [asyncpg](https://magicstack.github.io/asyncpg) (0.8+) * [PyICU](https://pypi.org/project/PyICU/) * [PyYaml](https://pyyaml.org/) (5.1+) * [datrie](https://github.com/pytries/datrie) +These will be installed automatically, when using pip installation. + +When using legacy CMake-based installation: + + * [cmake](https://cmake.org/) + * [expat](https://libexpat.github.io/) + * [proj](https://proj.org/) + * [bzip2](http://www.bzip.org/) + * [zlib](https://www.zlib.net/) + * [ICU](http://site.icu-project.org/) + * [nlohmann/json](https://json.nlohmann.me/) + * [Boost libraries](https://www.boost.org/), including system and file system + * PostgreSQL client libraries + * a recent C++ compiler (gcc 5+ or Clang 3.8+) + For running continuous updates: * [pyosmium](https://osmcode.org/pyosmium/) @@ -87,7 +95,7 @@ Take into account that the OSM database is growing fast. Fast disks are essential. Using NVME disks is recommended. Even on a well configured machine the import of a full planet takes -around 2 days. When using traditional SSDs, 4-5 days are more realistic. +around 2.5 days. When using traditional SSDs, 4-5 days are more realistic. ## Tuning the PostgreSQL database @@ -99,14 +107,16 @@ your `postgresql.conf` file. maintenance_work_mem = (10GB) autovacuum_work_mem = 2GB work_mem = (50MB) - effective_cache_size = (24GB) synchronous_commit = off max_wal_size = 1GB - checkpoint_timeout = 10min + checkpoint_timeout = 60min checkpoint_completion_target = 0.9 + random_page_cost = 1.0 + wal_level = minimal + max_wal_senders = 0 The numbers in brackets behind some parameters seem to work fine for -64GB RAM machine. Adjust to your setup. A higher number for `max_wal_size` +128GB RAM machine. Adjust to your setup. A higher number for `max_wal_size` means that PostgreSQL needs to run checkpoints less often but it does require the additional space on your disk. @@ -143,6 +153,18 @@ wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/countr ### Building Nominatim +#### Building the latest development version with pip + +To install Nominatim directly from the source tree, run: + + pip install packaging/nominatim-{core,db,api} + +#### Building in legacy CMake mode + +!!! warning + Installing Nominatim through CMake is now deprecated. The infrastructure + will be removed in Nominatim 5.0. Please switch to pip installation. + The code must be built in a separate directory. Create the directory and change into it. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 74465d1a..cb4ce9de 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -58,8 +58,8 @@ nav: - 'Testing' : 'develop/Testing.md' - 'External Data Sources': 'develop/data-sources.md' - 'Appendix': - - 'Installation on Ubuntu 20' : 'appendix/Install-on-Ubuntu-20.md' - 'Installation on Ubuntu 22' : 'appendix/Install-on-Ubuntu-22.md' + - 'Installation on Ubuntu 24' : 'appendix/Install-on-Ubuntu-24.md' markdown_extensions: - codehilite - admonition diff --git a/packaging/nominatim-db/pyproject.toml b/packaging/nominatim-db/pyproject.toml index 3e4a9a4f..312f1c81 100644 --- a/packaging/nominatim-db/pyproject.toml +++ b/packaging/nominatim-db/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ ] dependencies = [ "nominatim-core", - "psycopg2", + "psycopg2-binary", "psutil", "PyICU" ] diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh index ae0c0dea..9d8642a6 100755 --- a/vagrant/Install-on-Ubuntu-22.sh +++ b/vagrant/Install-on-Ubuntu-22.sh @@ -178,7 +178,7 @@ Environment="PYTHONPATH=/usr/local/lib/nominatim/lib-python/" User=www-data Group=www-data WorkingDirectory=$USERHOME/nominatim-project -ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim.server.falcon.server:run_wsgi +ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi ExecReload=/bin/kill -s HUP \$MAINPID StandardOutput=append:/var/log/gunicorn-nominatim.log StandardError=inherit diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-24.sh old mode 100755 new mode 100644 similarity index 80% rename from vagrant/Install-on-Ubuntu-20.sh rename to vagrant/Install-on-Ubuntu-24.sh index 57361ec7..3411c2ba --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-24.sh @@ -6,12 +6,12 @@ export APT_LISTCHANGES_FRONTEND=none #DOCS: export DEBIAN_FRONTEND=noninteractive #DOCS: # *Note:* these installation instructions are also available in executable -# form for use with vagrant under vagrant/Install-on-Ubuntu-20.sh. +# form for use with vagrant under vagrant/Install-on-Ubuntu-24.sh. # # Installing the Required Software # ================================ # -# These instructions expect that you have a freshly installed Ubuntu 20.04. +# These instructions expect that you have a freshly installed Ubuntu 24.04. # # Make sure all packages are up-to-date by running: # @@ -20,19 +20,8 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: # Now you can install all packages needed for Nominatim: - sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \ - libboost-filesystem-dev libexpat1-dev zlib1g-dev \ - libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \ - nlohmann-json3-dev postgresql-12-postgis-3 \ - postgresql-contrib-12 postgresql-12-postgis-3-scripts \ - libicu-dev python3-dotenv \ - python3-psycopg2 python3-psutil python3-jinja2 python3-pip \ - python3-icu python3-datrie python3-yaml git - -# Some of the Python packages that come with Ubuntu 20.04 are too old, so -# install the latest version from pip: - - pip3 install --user sqlalchemy asyncpg + sudo apt-get install -y osm2pgsql postgresql-postgis postgresql-postgis-scripts \ + pkg-config libicu-dev virtualenv git # @@ -78,14 +67,14 @@ fi #DOCS: # --------------------- # # Tune the postgresql configuration, which is located in -# `/etc/postgresql/12/main/postgresql.conf`. See section *Tuning the PostgreSQL database* +# `/etc/postgresql/14/main/postgresql.conf`. See section *Tuning the PostgreSQL database* # in [the installation page](../admin/Installation.md#tuning-the-postgresql-database) # for the parameters to change. # # Restart the postgresql service after updating this config file. if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: - sudo pg_ctlcluster 12 main start #DOCS: + sudo pg_ctlcluster 16 main start #DOCS: else #DOCS: sudo systemctl restart postgresql fi #DOCS: @@ -109,7 +98,7 @@ fi #DOCS: # if [ "x$1" == "xyes" ]; then #DOCS: :::sh cd $USERHOME - git clone --recursive https://github.com/openstreetmap/Nominatim.git + git clone https://github.com/openstreetmap/Nominatim.git cd Nominatim else #DOCS: cd $USERHOME/Nominatim #DOCS: @@ -122,32 +111,39 @@ if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz fi #DOCS: -# The code must be built in a separate directory. Create this directory, -# then configure and build Nominatim in there: +# Nominatim should be installed in a separate Python virtual environment. +# Create the virtual environment: + + virtualenv $USERHOME/nominatim-venv + +# Now install Nominatim using pip: + + cd $USERHOME/Nominatim + $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-{core,db} - mkdir $USERHOME/build - cd $USERHOME/build - cmake $USERHOME/Nominatim - make - sudo make install +# Nominatim is now ready to use. The nominatim binary is available at +# `$USERHOME/venv/bin/nominatim`. If you want to have 'nominatim' in your +# path, simply activate the virtual environment: -# Nominatim is now ready to use. You can continue with + + +# You can continue with # [importing a database from OSM data](../admin/Import.md). If you want to set up # the API frontend first, continue reading. # # Setting up the Python frontend # ============================== # -# Some of the Python packages in Ubuntu are too old. Therefore run the -# frontend from a Python virtualenv with current packages. +# The Python frontend is contained in the nominatim-api package. To run +# the API as a webservice, you also need falcon with uvicorn/gunicorn to +# serve the API. # -# To set up the virtualenv, run: +# To install all packages, run: #DOCS:```sh -sudo apt-get install -y virtualenv -virtualenv $USERHOME/nominatim-venv -$USERHOME/nominatim-venv/bin/pip install SQLAlchemy PyICU psycopg[binary] \ - psycopg2-binary python-dotenv PyYAML falcon uvicorn gunicorn +$USERHOME/nominatim-venv/bin/pip install psycopg[binary] falcon uvicorn gunicorn +cd $USERHOME/Nominatim +$USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api #DOCS:``` # Next you need to create a systemd job that runs Nominatim on gunicorn. @@ -182,7 +178,7 @@ Environment="PYTHONPATH=/usr/local/lib/nominatim/lib-python/" User=www-data Group=www-data WorkingDirectory=$USERHOME/nominatim-project -ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim.server.falcon.server:run_wsgi +ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi ExecReload=/bin/kill -s HUP \$MAINPID StandardOutput=append:/var/log/gunicorn-nominatim.log StandardError=inherit @@ -204,7 +200,6 @@ if [ "x$NOSYSTEMD" != "xyes" ]; then #DOCS: sudo systemctl enable nominatim.service fi #DOCS: - # Setting up a webserver # ====================== # @@ -216,23 +211,25 @@ fi #DOCS: # [during the import process](../admin/Import.md#creating-the-project-directory) # Already create the project directory itself now: - mkdir $USERHOME/nominatim-project - +# # Option 1: Using Apache # ---------------------- # if [ "x$2" == "xinstall-apache" ]; then #DOCS: +# # First install apache itself and enable the proxy module: sudo apt-get install -y apache2 sudo a2enmod proxy_http +# # To set up proxying for Apache add the following configuration: #DOCS:```sh sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF + ProxyPass /nominatim "unix:/run/nominatim.sock|http://localhost/" EOFAPACHECONF #DOCS:``` @@ -241,7 +238,10 @@ EOFAPACHECONF # Then enable the configuration and restart apache # - sudo a2enconf nominatim +#DOCS:```sh +sudo a2enconf nominatim +#DOCS:``` + if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: sudo apache2ctl start #DOCS: else #DOCS: @@ -262,6 +262,7 @@ if [ "x$2" == "xinstall-nginx" ]; then #DOCS: sudo apt-get install -y nginx + # Then create a Nginx configuration to forward http requests to that socket. #DOCS:```sh @@ -293,8 +294,6 @@ else #DOCS: sudo systemctl restart nginx fi #DOCS: -# The Nominatim API is now available at `http://localhost/`. - - +# The Nominatim API is now available at `http://localhost/nominatim/`. fi #DOCS: -- 2.39.5