From: Sarah Hoffmann Date: Tue, 19 Nov 2024 09:02:53 +0000 (+0100) Subject: Merge pull request #3592 from lonvia/remove-cmake X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/c172ca8c6c060f188ad626bca631a7c2cd908e7d?hp=02364ce6c8334b314ea543becbb93381e6c1c1ee Merge pull request #3592 from lonvia/remove-cmake Remove cmake building and bundled osm2pgsql --- diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index d05fd319..85eef53f 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -1,18 +1,10 @@ name: 'Build Nominatim' inputs: - flavour: - description: 'Version of Ubuntu to install on' + dependencies: + description: 'Where to install dependencies from (pip/apt)' required: false - default: 'ubuntu-20' - cmake-args: - description: 'Additional options to hand to cmake' - required: false - default: '' - lua: - description: 'Version of Lua to use' - required: false - default: '5.3' + default: 'pip' runs: using: "composite" @@ -23,30 +15,30 @@ runs: sudo rm -rf /opt/hostedtoolcache/go /opt/hostedtoolcache/CodeQL /usr/lib/jvm /usr/local/share/chromium /usr/local/lib/android df -h shell: bash - - name: Install${{ matrix.flavour }} prerequisites + - name: Install general prerequisites + run: | + sudo apt-get install -y -qq libspatialite-dev libsqlite3-mod-spatialite libicu-dev virtualenv python3-dev osm2pgsql + shell: bash + + - name: Install prerequisites from apt run: | - sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson nlohmann-json3-dev libspatialite-dev libsqlite3-mod-spatialite - if [ "$FLAVOUR" == "oldstuff" ]; then - pip3 install MarkupSafe==2.0.1 python-dotenv jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4.31 psycopg==3.1.7 datrie asyncpg aiosqlite - else - sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-dotenv python3-yaml - pip3 install sqlalchemy psycopg aiosqlite - fi + sudo apt-get install -y -qq python3-icu python3-datrie python3-jinja2 python3-psutil python3-dotenv python3-yaml python3-sqlalchemy python3-psycopg python3-asyncpg shell: bash - env: - FLAVOUR: ${{ inputs.flavour }} - CMAKE_ARGS: ${{ inputs.cmake-args }} - LUA_VERSION: ${{ inputs.lua }} + if: inputs.dependencies == 'apt' - - name: Configure - run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim + - name: Setup virtual environment (for pip) + run: | + virtualenv venv + ./venv/bin/pip install -U pip shell: bash - env: - CMAKE_ARGS: ${{ inputs.cmake-args }} + if: inputs.dependencies == 'pip' - - name: Build + - name: Setup virtual environment (for apt) run: | - make -j2 all - sudo make install + virtualenv venv --system-site-packages + shell: bash + if: inputs.dependencies == 'apt' + + - name: Build nominatim + run: ./venv/bin/pip install Nominatim/packaging/nominatim-{api,db} shell: bash - working-directory: build diff --git a/.github/actions/setup-postgresql/action.yml b/.github/actions/setup-postgresql/action.yml index 416ced5f..331d094a 100644 --- a/.github/actions/setup-postgresql/action.yml +++ b/.github/actions/setup-postgresql/action.yml @@ -4,9 +4,6 @@ inputs: postgresql-version: description: 'Version of PostgreSQL to install' required: true - postgis-version: - description: 'Version of Postgis to install' - required: true runs: using: "composite" @@ -24,11 +21,10 @@ runs: - name: Install PostgreSQL run: | - sudo apt-get install -y -qq --no-install-suggests --no-install-recommends postgresql-client-${PGVER} postgresql-${PGVER}-postgis-${POSTGISVER} postgresql-${PGVER}-postgis-${POSTGISVER}-scripts postgresql-contrib-${PGVER} postgresql-${PGVER} + sudo apt-get install -y -qq --no-install-suggests --no-install-recommends postgresql-client-${PGVER} postgresql-${PGVER}-postgis-3 postgresql-${PGVER}-postgis-3-scripts postgresql-contrib-${PGVER} postgresql-${PGVER} shell: bash env: PGVER: ${{ inputs.postgresql-version }} - POSTGISVER: ${{ inputs.postgis-version }} - name: Adapt postgresql configuration run: | diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 293743df..8b38d3dc 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -42,13 +42,13 @@ jobs: - flavour: ubuntu-20 ubuntu: 20 postgresql: 12 - postgis: 3 lua: '5.1' + dependencies: pip - flavour: ubuntu-24 ubuntu: 24 postgresql: 17 - postgis: 3 lua: '5.3' + dependencies: apt runs-on: ubuntu-${{ matrix.ubuntu }}.04 @@ -60,58 +60,67 @@ jobs: - name: Unpack Nominatim run: tar xf nominatim-src.tar.bz2 - - uses: actions/setup-python@v5 - with: - python-version: 3.7 - if: matrix.flavour == 'oldstuff' - - uses: ./Nominatim/.github/actions/setup-postgresql with: postgresql-version: ${{ matrix.postgresql }} - postgis-version: ${{ matrix.postgis }} - uses: ./Nominatim/.github/actions/build-nominatim with: - flavour: ${{ matrix.flavour }} - lua: ${{ matrix.lua }} + dependencies: ${{ matrix.dependencies }} - - name: Install test prerequisites - run: pip3 install behave==1.2.6 + - name: Compile osm2pgsql + run: | + sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev + mkdir osm2pgsql-build + cd osm2pgsql-build + git clone https://github.com/osm2pgsql-dev/osm2pgsql + mkdir build + cd build + cmake ../osm2pgsql + make + sudo make install + cd ../.. + rm -rf osm2pgsql-build + if: matrix.ubuntu == '20' + env: + LUA_VERSION: ${{ matrix.lua }} - name: Install test prerequisites - run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn + run: ./venv/bin/pip install behave==1.2.6 - - name: Install newer pytest-asyncio - run: pip3 install -U pytest-asyncio - if: matrix.flavour == 'ubuntu-20' + - name: Install test prerequisites (apt) + run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn python3-falcon python3-aiosqlite python3-pyosmium + if: matrix.dependencies == 'apt' - - name: Install Python webservers - run: pip3 install falcon starlette asgi_lifespan + - name: Install test prerequisites (pip) + run: ./venv/bin/pip install pytest-asyncio falcon starlette asgi_lifespan aiosqlite osmium uvicorn + if: matrix.dependencies == 'pip' - name: Install latest flake8 - run: pip3 install -U flake8 + run: ./venv/bin/pip install -U flake8 - name: Python linting - run: python3 -m flake8 src + run: ../venv/bin/python -m flake8 src working-directory: Nominatim + - name: Install mypy and typechecking info + run: ./venv/bin/pip install -U mypy types-PyYAML types-jinja2 types-psutil types-requests types-ujson types-Pygments typing-extensions + if: matrix.dependencies == 'pip' + + - name: Python static typechecking + run: ../venv/bin/python -m mypy --strict --python-version 3.8 src + working-directory: Nominatim + if: matrix.dependencies == 'pip' + - name: Python unit tests - run: python3 -m pytest test/python + run: ../venv/bin/python -m pytest test/python working-directory: Nominatim - name: BDD tests run: | - export PATH=$GITHUB_WORKSPACE/build/osm2pgsql:$PATH - python3 -m behave -DREMOVE_TEMPLATE=1 --format=progress3 + ../../../venv/bin/python -m behave -DREMOVE_TEMPLATE=1 --format=progress3 working-directory: Nominatim/test/bdd - - name: Install mypy and typechecking info - run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson types-Pygments typing-extensions - - - name: Python static typechecking - run: python3 -m mypy --strict --python-version 3.8 src - working-directory: Nominatim - install: runs-on: ubuntu-latest needs: create-archive @@ -186,12 +195,10 @@ jobs: - name: Add nominatim to path run: | sudo ln -s /home/nominatim/nominatim-venv/bin/nominatim /usr/local/bin/nominatim - if: matrix.ubuntu == 24 - name: Need lua binary run: | sudo apt-get install -y lua5.4 lua-dkjson - if: matrix.ubuntu == 24 - name: Print version run: nominatim --version @@ -221,19 +228,9 @@ jobs: run: nominatim admin --warm working-directory: /home/nominatim/nominatim-project - - name: Prepare update (Ubuntu) - run: apt-get install -y python3-pip - shell: bash - - - name: Install osmium (Ubuntu 22) - run: | - pip3 install --user osmium - if: matrix.ubuntu == 22 - - - name: Install osmium (Ubuntu 24) + - name: Install osmium run: | /home/nominatim/nominatim-venv/bin/pip install osmium - if: matrix.ubuntu == 24 - name: Run update run: | @@ -260,7 +257,7 @@ jobs: working-directory: /home/nominatim/nominatim-project install-no-superuser: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: create-archive steps: @@ -274,12 +271,8 @@ jobs: - uses: ./Nominatim/.github/actions/setup-postgresql with: postgresql-version: 16 - postgis-version: 3 - uses: ./Nominatim/.github/actions/build-nominatim - with: - flavour: ubuntu-22 - lua: 5.3 - name: Prepare import environment run: | @@ -288,7 +281,7 @@ jobs: - name: Prepare Database run: | - nominatim import --prepare-database + ./venv/bin/nominatim import --prepare-database - name: Create import user run: | @@ -298,10 +291,10 @@ jobs: - name: Run import run: | - NOMINATIM_DATABASE_DSN="pgsql:host=127.0.0.1;dbname=nominatim;user=osm-import;password=osm-import" nominatim import --continue import-from-file --osm-file test.pbf + NOMINATIM_DATABASE_DSN="pgsql:host=127.0.0.1;dbname=nominatim;user=osm-import;password=osm-import" ./venv/bin/nominatim import --continue import-from-file --osm-file test.pbf - name: Check full import - run: nominatim admin --check-database + run: ./venv/bin/nominatim admin --check-database migrate: runs-on: ubuntu-24.04 @@ -318,7 +311,6 @@ jobs: - uses: ./Nominatim/.github/actions/setup-postgresql with: postgresql-version: 17 - postgis-version: 3 - name: Install Python dependencies run: | diff --git a/.gitmodules b/.gitmodules index 71b82eab..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +0,0 @@ -[submodule "osm2pgsql"] - path = osm2pgsql - url = https://github.com/openstreetmap/osm2pgsql.git - ignore = dirty diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index e31362fe..00000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,216 +0,0 @@ -#----------------------------------------------------------------------------- -# -# CMake Config -# -# Nominatim -# -#----------------------------------------------------------------------------- - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") - - -#----------------------------------------------------------------------------- -# -# Project version -# -#----------------------------------------------------------------------------- - -project(nominatim) - -set(NOMINATIM_VERSION_MAJOR 4) -set(NOMINATIM_VERSION_MINOR 5) -set(NOMINATIM_VERSION_PATCH 0) - -set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}") - -add_definitions(-DNOMINATIM_VERSION="${NOMINATIM_VERSION}") - -# Setting GIT_HASH -find_package(Git) -if (GIT_FOUND) - execute_process( - COMMAND "${GIT_EXECUTABLE}" log -1 --format=%h - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - OUTPUT_VARIABLE GIT_HASH - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) -endif() - -#----------------------------------------------------------------------------- -# Configuration -#----------------------------------------------------------------------------- - -set(BUILD_IMPORTER on CACHE BOOL "Build everything for importing/updating the database") -set(BUILD_API on CACHE BOOL "Build everything for the API server") -set(BUILD_TESTS on CACHE BOOL "Build test suite") -set(BUILD_OSM2PGSQL on CACHE BOOL "Build osm2pgsql (expert only)") -set(INSTALL_MUNIN_PLUGINS on CACHE BOOL "Install Munin plugins for supervising Nominatim") - -#----------------------------------------------------------------------------- -# osm2pgsql (imports/updates only) -#----------------------------------------------------------------------------- - -if (BUILD_IMPORTER AND BUILD_OSM2PGSQL) - if (NOT EXISTS "${CMAKE_SOURCE_DIR}/osm2pgsql/CMakeLists.txt") - message(FATAL_ERROR "The osm2pgsql directory is empty.\ - Did you forget to check out Nominatim recursively?\ - \nTry updating submodules with: git submodule update --init") - endif() - set(BUILD_TESTS_SAVED "${BUILD_TESTS}") - set(BUILD_TESTS off) - add_subdirectory(osm2pgsql) - set(BUILD_TESTS ${BUILD_TESTS_SAVED}) -endif() - - -#----------------------------------------------------------------------------- -# python (imports/updates only) -#----------------------------------------------------------------------------- - -if (BUILD_IMPORTER OR BUILD_API) - find_package(PythonInterp 3.7 REQUIRED) -endif() - -#----------------------------------------------------------------------------- -# import scripts and utilities (importer only) -#----------------------------------------------------------------------------- - -if (BUILD_IMPORTER) - find_file(COUNTRY_GRID_FILE country_osm_grid.sql.gz - PATHS ${PROJECT_SOURCE_DIR}/data - NO_DEFAULT_PATH - DOC "Location of the country grid file." - ) - - if (NOT COUNTRY_GRID_FILE) - message(FATAL_ERROR "\nYou need to download the country_osm_grid first:\n" - " wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz") - endif() - - configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl - ${PROJECT_BINARY_DIR}/nominatim) -endif() - -#----------------------------------------------------------------------------- -# Tests -#----------------------------------------------------------------------------- - -if (BUILD_TESTS) - include(CTest) - - set(TEST_BDD db osm2pgsql api) - - find_program(PYTHON_BEHAVE behave) - find_program(PYLINT NAMES pylint3 pylint) - find_program(PYTEST NAMES pytest py.test-3 py.test) - - if (PYTHON_BEHAVE) - message(STATUS "Using Python behave binary ${PYTHON_BEHAVE}") - foreach (test ${TEST_BDD}) - add_test(NAME bdd_${test} - COMMAND ${PYTHON_BEHAVE} ${test} - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/bdd) - set_tests_properties(bdd_${test} - PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}") - endforeach() - else() - message(WARNING "behave not found. BDD tests disabled." ) - endif() - - if (PYLINT) - message(STATUS "Using pylint binary ${PYLINT}") - add_test(NAME pylint - COMMAND ${PYLINT} nominatim - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) - else() - message(WARNING "pylint not found. Python linting tests disabled.") - endif() - - if (PYTEST) - message(STATUS "Using pytest binary ${PYTEST}") - add_test(NAME pytest - COMMAND ${PYTEST} test/python - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) - else() - message(WARNING "pytest not found. Python tests disabled." ) - endif() -endif() - -#----------------------------------------------------------------------------- -# Installation -#----------------------------------------------------------------------------- - - -include(GNUInstallDirs) -set(NOMINATIM_DATADIR ${CMAKE_INSTALL_FULL_DATADIR}/${PROJECT_NAME}) -set(NOMINATIM_LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR}/${PROJECT_NAME}) -set(NOMINATIM_CONFIGDIR ${CMAKE_INSTALL_FULL_SYSCONFDIR}/${PROJECT_NAME}) -set(NOMINATIM_MUNINDIR ${CMAKE_INSTALL_FULL_DATADIR}/munin/plugins) - -if (BUILD_IMPORTER) - configure_file(${PROJECT_SOURCE_DIR}/cmake/tool-installed.tmpl installed.bin) - install(PROGRAMS ${PROJECT_BINARY_DIR}/installed.bin - DESTINATION ${CMAKE_INSTALL_BINDIR} - RENAME nominatim) - - configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py-no-php.tmpl paths-py.installed) - - foreach (submodule nominatim_db nominatim_api) - install(DIRECTORY src/${submodule} - DESTINATION ${NOMINATIM_LIBDIR}/lib-python - FILES_MATCHING PATTERN "*.py" - PATTERN "paths.py" EXCLUDE - PATTERN __pycache__ EXCLUDE) - install(FILES ${PROJECT_BINARY_DIR}/paths-py.installed - DESTINATION ${NOMINATIM_LIBDIR}/lib-python/${submodule} - RENAME paths.py) - endforeach() - - install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR}) - - install(FILES ${COUNTRY_GRID_FILE} - data/words.sql - DESTINATION ${NOMINATIM_DATADIR}) -endif() - -if (BUILD_OSM2PGSQL) - if (${CMAKE_VERSION} VERSION_LESS 3.13) - # Installation of subdirectory targets was only introduced in 3.13. - # So just copy the osm2pgsql file for older versions. - install(PROGRAMS ${PROJECT_BINARY_DIR}/osm2pgsql/osm2pgsql - DESTINATION ${NOMINATIM_LIBDIR}) - else() - install(TARGETS osm2pgsql RUNTIME DESTINATION ${NOMINATIM_LIBDIR}) - endif() -endif() - -install(FILES settings/env.defaults - settings/address-levels.json - settings/phrase-settings.json - settings/import-admin.lua - settings/import-street.lua - settings/import-address.lua - settings/import-full.lua - settings/import-extratags.lua - settings/flex-base.lua - settings/icu_tokenizer.yaml - settings/country_settings.yaml - DESTINATION ${NOMINATIM_CONFIGDIR}) - -install(DIRECTORY settings/icu-rules - DESTINATION ${NOMINATIM_CONFIGDIR}) -install(DIRECTORY settings/country-names - DESTINATION ${NOMINATIM_CONFIGDIR}) - -if (INSTALL_MUNIN_PLUGINS) - install(FILES munin/nominatim_importlag - munin/nominatim_query_speed - munin/nominatim_requests - DESTINATION ${NOMINATIM_MUNINDIR}) -endif() - -message(WARNING "Building with CMake is deprecated and will be removed in Nominatim 5.0." - "Use Nominatim pip packages instead.\n" - "See https://nominatim.org/release-docs/develop/admin/Installation/#downloading-and-building-nominatim") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 78c4dd67..b9bf2920 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,13 +92,14 @@ Checklist for releases: * [ ] complete `docs/admin/Migration.md` * [ ] update EOL dates in `SECURITY.md` * [ ] commit and make sure CI tests pass +* [ ] update OSMF production repo and release new version -post1 there * [ ] test migration * download, build and import previous version * migrate using master version * run updates using master version * [ ] prepare tarball: - * `git clone --recursive https://github.com/osm-search/Nominatim` (switch to right branch!) - * `rm -r .git* osm2pgsql/.git*` + * `git clone https://github.com/osm-search/Nominatim` (switch to right branch!) + * `rm -r .git*` * copy country data into `data/` * add version to base directory and package * [ ] upload tarball to https://nominatim.org diff --git a/README.md b/README.md index d130074e..3b0f328a 100644 --- a/README.md +++ b/README.md @@ -27,21 +27,21 @@ can be found at nominatim.org as well. A quick summary of the necessary steps: -1. Compile Nominatim: +1. Create a Python virtualenv and install the packages: - mkdir build - cd build - cmake .. - make - sudo make install + python3 -m venv nominatim-venv + ./nominatim-venv/bin/pip install packaging/nominatim-{api,db} 2. Create a project directory, get OSM data and import: mkdir nominatim-project cd nominatim-project - nominatim import --osm-file + ../nominatim-venv/bin/nominatim import --osm-file -3. Point your webserver to the nominatim-project/website directory. +3. Start the webserver: + + ./nominatim-venv/bin/pip install uvicorn falcon + ../nominatim-venv/bin/nominatim serve License diff --git a/VAGRANT.md b/VAGRANT.md index 819c6071..4ff9d4bd 100644 --- a/VAGRANT.md +++ b/VAGRANT.md @@ -1,6 +1,6 @@ # Install Nominatim in a virtual machine for development and testing -This document describes how you can install Nominatim inside a Ubuntu 22 +This document describes how you can install Nominatim inside a Ubuntu 24 virtual machine on your desktop/laptop (host machine). The goal is to give you a development environment to easily edit code and run the test suite without affecting the rest of your system. @@ -15,29 +15,22 @@ is. 2. [Vagrant](https://www.vagrantup.com/downloads.html) -3. Nominatim - - git clone --recursive https://github.com/openstreetmap/Nominatim.git - - If you forgot `--recursive`, it you can later load the submodules using - - git submodule init - git submodule update - +3. Nominatim + git clone https://github.com/openstreetmap/Nominatim.git ## Installation 1. Start the virtual machine - vagrant up ubuntu + vagrant up ubuntu24-nginx 2. Log into the virtual machine - vagrant ssh ubuntu + vagrant ssh ubuntu24-nginx 3. Import a small country (Monaco) - + See the FAQ how to skip this step and point Nominatim to an existing database. ``` @@ -61,73 +54,22 @@ see Nominatim in action on [localhost:8089](http://localhost:8089/nominatim/). You edit code on your host machine in any editor you like. There is no need to restart any software: just refresh your browser window. -Note that the webserver uses files from the /build directory. If you change -files in Nominatim/website or Nominatim/utils for example you first need to -copy them into the /build directory by running the `cmake` step from the -installation. - -PHP errors are written to `/var/log/apache2/error.log`. - -With `echo` and `var_dump()` you write into the output (HTML/XML/JSON) when -you either add `&debug=1` to the URL. +Use the functions of the `log()` object to create temporary debug output. +Add `&debug=1` to the URL to see the output. In the Python BDD test you can use `logger.info()` for temporary debug statements. - - -## Running unit tests - - cd ~/Nominatim/tests/php - phpunit ./ - - -## Running PHP code style tests - - cd ~/Nominatim - phpcs --colors . - - -## Running functional tests - -Tests in `test/bdd/db` and `test/bdd/osm2pgsql` have to pass 100%. Other -tests might require full planet-wide data. Sadly even if you have your own -planet-wide data there will be enough differences to the openstreetmap.org -installation to cause false positives in the other tests (see FAQ). - -To run the full test suite - - cd ~/Nominatim/test/bdd - behave -DBUILDDIR=/home/vagrant/build/ db osm2pgsql - -To run a single file - - behave -DBUILDDIR=/home/vagrant/build/ api/lookup/simple.feature - -Or a single test by line number - - behave -DBUILDDIR=/home/vagrant/build/ api/lookup/simple.feature:34 - -To run specific groups of tests you can add tags just before the `Scenario line`, e.g. - - @bug-34 - Scenario: address lookup for non-existing or invalid node, way, relation - -and then - - behave -DBUILDDIR=/home/vagrant/build/ --tags @bug-34 - - - - +For more information on running tests, see +https://nominatim.org/release-docs/develop/develop/Testing/ ## FAQ ##### Will it run on Windows? -Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need a 64bit -version of Windows. +Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need +a 64bit version of Windows. ##### Will it run on Apple Silicon? @@ -136,11 +78,11 @@ There is no free/open source version of Parallels. ##### Why Monaco, can I use another country? -Of course! The Monaco import takes less than 30 minutes and works with 2GB RAM. +Of course! The Monaco import takes less than 10 minutes and works with 2GB RAM. ##### Will the results be the same as those from nominatim.openstreetmap.org? -No. Long running Nominatim installations will differ once new import features (or +No. Long-running Nominatim installations will differ once new import features (or bug fixes) get added since those usually only get applied to new/changed data. Also this document skips the optional Wikipedia data import which affects ranking @@ -188,7 +130,3 @@ e.g. `psql --host localhost --port 9999 nominatim_it` Yes. It's possible to start the virtual machine on [Amazon AWS (plugin)](https://github.com/mitchellh/vagrant-aws) or [DigitalOcean (plugin)](https://github.com/smdahlen/vagrant-digitalocean). - - - - diff --git a/cmake/paths-py-no-php.tmpl b/cmake/paths-py-no-php.tmpl deleted file mode 100644 index a95cb664..00000000 --- a/cmake/paths-py-no-php.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2022 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Path settings for extra data used by Nominatim (installed version). -""" -from pathlib import Path - -SQLLIB_DIR = (Path('@NOMINATIM_LIBDIR@') / 'lib-sql').resolve() -DATA_DIR = Path('@NOMINATIM_DATADIR@').resolve() -CONFIG_DIR = Path('@NOMINATIM_CONFIGDIR@').resolve() diff --git a/cmake/tool-installed.tmpl b/cmake/tool-installed.tmpl deleted file mode 100644 index 915f1d69..00000000 --- a/cmake/tool-installed.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 -import sys -import os - -sys.path.insert(1, '@NOMINATIM_LIBDIR@/lib-python') - -from nominatim_db import cli -from nominatim_db import version - -version.GIT_COMMIT_HASH = '@GIT_HASH@' - -exit(cli.nominatim(module_dir='@NOMINATIM_LIBDIR@/module', - osm2pgsql_path='@NOMINATIM_LIBDIR@/osm2pgsql')) diff --git a/cmake/tool.tmpl b/cmake/tool.tmpl deleted file mode 100755 index 4053ca35..00000000 --- a/cmake/tool.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 -import sys -import os - -sys.path.insert(1, '@CMAKE_SOURCE_DIR@/src') - -from nominatim_db import cli -from nominatim_db import version - -version.GIT_COMMIT_HASH = '@GIT_HASH@' - -exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module', - osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql')) diff --git a/docs/admin/Faq.md b/docs/admin/Faq.md index efcd5741..1d0252d3 100644 --- a/docs/admin/Faq.md +++ b/docs/admin/Faq.md @@ -37,31 +37,6 @@ nominatim import --continue indexing Otherwise it's best to start the full setup from the beginning. - -### nominatim.so version mismatch - -When running the import you may get a version mismatch: -`COPY_END for place failed: ERROR: incompatible library "/srv/Nominatim/nominatim/build/module/nominatim.so": version mismatch` - -pg_config seems to use bad includes sometimes when multiple versions -of PostgreSQL are available in the system. Make sure you remove the -server development libraries (`postgresql-server-dev-13` on Ubuntu) -and recompile (`cmake .. && make`). - - -### I see the error "ERROR: permission denied for language c" - -`nominatim.so`, written in C, is required to be installed on the database -server. Some managed database (cloud) services like Amazon RDS do not allow -this. There is currently no work-around other than installing a database -on a non-managed machine. - - -### I see the error: "function transliteration(text) does not exist" - -Reinstall the nominatim functions with `nominatim refresh --functions` -and check for any errors, e.g. a missing `nominatim.so` file. - ### I see the error: "ERROR: mmap (remap) failed" This may be a simple out-of-memory error. Try reducing the memory used @@ -103,39 +78,6 @@ for default Ubuntu operating system for example it's `www-data`. GRANT SELECT ON ALL TABLES IN SCHEMA public TO "www-data"; ``` -### Website reports "Could not load library "nominatim.so" - -Example error message - -``` - SELECT make_standard_name('3039 E MEADOWLARK LN') [nativecode=ERROR: could not - load library "/srv/nominatim/Nominatim-3.1.0/build/module/nominatim.so": - /srv/nominatim/Nominatim-3.1.0/build/module/nominatim.so: cannot open shared - object file: Permission denied - CONTEXT: PL/pgSQL function make_standard_name(text) line 5 at assignment] -``` - -The PostgreSQL database, i.e. user `postgres`, needs to have access to that file. - -The permission need to be read & executable by everybody, but not writeable -by everybody, e.g. - -``` - -rwxr-xr-x 1 nominatim nominatim 297984 build/module/nominatim.so -``` - -Try `chmod a+r nominatim.so; chmod a+x nominatim.so`. - -When you recently updated your operating system, updated PostgreSQL to -a new version or moved files (e.g. the build directory) you should -recreate `nominatim.so`. Try - -``` - cd build - rm -r module/ - cmake $main_Nominatim_path && make -``` - ### Setup fails with "DB Error: extension not found" Make sure you have the PostgreSQL extensions "hstore" and "postgis" installed. @@ -172,4 +114,6 @@ for more information. ### Can I import negative OSM ids into Nominatim? -See [this question of Stackoverflow](https://help.openstreetmap.org/questions/64662/nominatim-flatnode-with-negative-id). +No, negative IDs are no longer supported by osm2pgsql. You can use +large 64-bit IDs that are guaranteed not to clash with OSM IDs. However, +you will not able to use a flatnode file with them. diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index 3db16c3b..dadbe460 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -41,19 +41,6 @@ Furthermore the following Python libraries are required: These will be installed automatically when using pip installation. -When using legacy CMake-based installation: - - * [cmake](https://cmake.org/) - * [expat](https://libexpat.github.io/) - * [proj](https://proj.org/) - * [bzip2](http://www.bzip.org/) - * [zlib](https://www.zlib.net/) - * [ICU](http://site.icu-project.org/) - * [nlohmann/json](https://json.nlohmann.me/) - * [Boost libraries](https://www.boost.org/), including system and file system - * PostgreSQL client libraries - * a recent C++ compiler (gcc 5+ or Clang 3.8+) - For running continuous updates: * [pyosmium](https://osmcode.org/pyosmium/) @@ -124,11 +111,10 @@ The release contains all necessary files. Just unpack it. ### Downloading the latest development version -If you want to install latest development version from github, make sure to -also check out the osm2pgsql subproject: +If you want to install latest development version from github: ``` -git clone --recursive https://github.com/openstreetmap/Nominatim.git +git clone https://github.com/osm-search/Nominatim.git ``` The development version does not include the country grid. Download it separately: @@ -139,8 +125,6 @@ wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/countr ### Building Nominatim -#### Building the latest development version with pip - Nominatim is easiest to run from its own virtual environment. To create one, run: sudo apt-get install virtualenv @@ -150,36 +134,5 @@ To install Nominatim directly from the source tree into the virtual environment, /srv/nominatim-venv/bin/pip install packaging/nominatim-{db,api} -#### Building in legacy CMake mode - -!!! warning - Installing Nominatim through CMake is now deprecated. The infrastructure - will be removed in Nominatim 5.0. Please switch to pip installation. - -The code must be built in a separate directory. Create the directory and -change into it. - -``` -mkdir build -cd build -``` - -Nominatim uses cmake and make for building. Assuming that you have created the -build at the same level as the Nominatim source directory run: - -``` -cmake ../Nominatim -make -sudo make install -``` - -Nominatim installs itself into `/usr/local` per default. To choose a different -installation directory add `-DCMAKE_INSTALL_PREFIX=` to the -cmake command. Make sure that the `bin` directory is available in your path -in that case, e.g. - -``` -export PATH=/bin:$PATH -``` Now continue with [importing the database](Import.md). diff --git a/osm2pgsql b/osm2pgsql deleted file mode 160000 index cf66989f..00000000 --- a/osm2pgsql +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cf66989fa2a47aa406f25c0be79e3b4e146284ee diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh index a1375831..2ad337bb 100755 --- a/vagrant/Install-on-Ubuntu-22.sh +++ b/vagrant/Install-on-Ubuntu-22.sh @@ -25,15 +25,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \ nlohmann-json3-dev postgresql-14-postgis-3 \ postgresql-contrib-14 postgresql-14-postgis-3-scripts \ - libicu-dev python3-dotenv \ - python3-pip python3-psutil python3-jinja2 \ - python3-sqlalchemy python3-asyncpg \ - python3-icu python3-datrie python3-yaml git - -# Some of the Python packages that come with Ubuntu 22.04 are too old, -# so install the latest version from pip: - - pip3 install --user psycopg[binary] + libicu-dev virtualenv git # # System Configuration @@ -109,7 +101,7 @@ fi #DOCS: # if [ "x$1" == "xyes" ]; then #DOCS: :::sh cd $USERHOME - git clone --recursive https://github.com/openstreetmap/Nominatim.git + git clone https://github.com/osm-search/Nominatim.git cd Nominatim else #DOCS: cd $USERHOME/Nominatim #DOCS: @@ -122,14 +114,31 @@ if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz fi #DOCS: -# The code must be built in a separate directory. Create this directory, -# then configure and build Nominatim in there: +# Nominatim needs osm2pgsql >= 1.8. The version that comes with Ubuntu is +# too old. Download and compile your own: - mkdir $USERHOME/build - cd $USERHOME/build - cmake $USERHOME/Nominatim + cd $USERHOME + git clone https://github.com/osm2pgsql-dev/osm2pgsql + mkdir osm2pgsql-build + cd osm2pgsql-build + cmake ../osm2pgsql make sudo make install + cd $USERHOME/Nominatim + +# Nominatim should be installed in a separate Python virtual environment. +# Create the virtual environment: + + virtualenv $USERHOME/nominatim-venv + +# We want the faster binary version pf psycopg, so install that: + + $USERHOME/nominatim-venv/bin/pip install psycopg[binary] + +# Now install Nominatim using pip: + + cd $USERHOME/Nominatim + $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-db # Nominatim is now ready to use. You can continue with # [importing a database from OSM data](../admin/Import.md). If you want to set up @@ -138,18 +147,19 @@ fi #DOCS: # Setting up the Python frontend # ============================== # -# Some of the Python packages in Ubuntu are too old. Therefore run the -# frontend from a Python virtualenv with current packages. +# The Python frontend is contained in the nominatim-api package. To run +# the API as a webservice, you also need falcon with uvicorn to serve the API. +# It is generally recommended to run falcon/uvicorn on top of gunicorn. # -# To set up the virtualenv, run: +# To install all packages, run: #DOCS:```sh -sudo apt-get install -y virtualenv -virtualenv $USERHOME/nominatim-venv -$USERHOME/nominatim-venv/bin/pip install SQLAlchemy PyICU psycopg[binary] \ - psycopg2-binary python-dotenv PyYAML falcon uvicorn gunicorn +$USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn +cd $USERHOME/Nominatim +$USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api #DOCS:``` + # Next you need to create a systemd job that runs Nominatim on gunicorn. # First create a systemd job that manages the socket file: @@ -178,14 +188,11 @@ Requires=nominatim.socket [Service] Type=simple -Environment="PYTHONPATH=/usr/local/lib/nominatim/lib-python/" User=www-data Group=www-data WorkingDirectory=$USERHOME/nominatim-project -ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi +ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker "nominatim_api.server.falcon.server:run_wsgi()" ExecReload=/bin/kill -s HUP \$MAINPID -StandardOutput=append:/var/log/gunicorn-nominatim.log -StandardError=inherit PrivateTmp=true TimeoutStopSec=5 KillMode=mixed diff --git a/vagrant/Install-on-Ubuntu-24.sh b/vagrant/Install-on-Ubuntu-24.sh index 35882a79..b15eb3d8 100755 --- a/vagrant/Install-on-Ubuntu-24.sh +++ b/vagrant/Install-on-Ubuntu-24.sh @@ -98,7 +98,7 @@ fi #DOCS: # if [ "x$1" == "xyes" ]; then #DOCS: :::sh cd $USERHOME - git clone https://github.com/openstreetmap/Nominatim.git + git clone https://github.com/osm-search/Nominatim.git cd Nominatim else #DOCS: cd $USERHOME/Nominatim #DOCS: @@ -116,6 +116,10 @@ fi #DOCS: virtualenv $USERHOME/nominatim-venv +# We want the faster binary version pf psycopg, so install that: + + $USERHOME/nominatim-venv/bin/pip install psycopg[binary] + # Now install Nominatim using pip: cd $USERHOME/Nominatim @@ -143,7 +147,7 @@ fi #DOCS: # To install all packages, run: #DOCS:```sh -$USERHOME/nominatim-venv/bin/pip install psycopg[binary] falcon uvicorn gunicorn +$USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn cd $USERHOME/Nominatim $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api #DOCS:``` @@ -179,10 +183,8 @@ Type=simple User=www-data Group=www-data WorkingDirectory=$USERHOME/nominatim-project -ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi +ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker "nominatim_api.server.falcon.server:run_wsgi()" ExecReload=/bin/kill -s HUP \$MAINPID -StandardOutput=append:/var/log/gunicorn-nominatim.log -StandardError=inherit PrivateTmp=true TimeoutStopSec=5 KillMode=mixed