From: Sarah Hoffmann Date: Mon, 11 Jul 2022 13:38:26 +0000 (+0200) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~100 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/f3c557bf684a0079e4bc54b622cc5d766f3a6b56?hp=3bcd32ca2006cb2184f290e18f4e054f8aca8bf4 Merge remote-tracking branch 'upstream/master' --- diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index 042166ad..c6ff7a31 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -14,6 +14,11 @@ runs: using: "composite" steps: + - name: Clean out the disk + run: | + sudo rm -rf /opt/hostedtoolcache/go /opt/hostedtoolcache/CodeQL /usr/lib/jvm /usr/local/share/chromium /usr/local/lib/android + df -h + shell: bash - name: Install prerequisites run: | sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4ce14f93..53b76a03 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -37,7 +37,7 @@ jobs: needs: create-archive strategy: matrix: - ubuntu: [18, 20] + ubuntu: [18, 20, 22] include: - ubuntu: 18 postgresql: 9.6 @@ -49,6 +49,11 @@ jobs: postgis: 3 pytest: py.test-3 php: 7.4 + - ubuntu: 22 + postgresql: 14 + postgis: 3 + pytest: py.test-3 + php: 8.1 runs-on: ubuntu-${{ matrix.ubuntu }}.04 @@ -65,6 +70,7 @@ jobs: with: php-version: ${{ matrix.php }} tools: phpunit, phpcs, composer + ini-values: opcache.jit=disable - uses: actions/setup-python@v2 with: @@ -85,8 +91,12 @@ jobs: if: matrix.ubuntu == 20 - name: Install test prerequsites - run: pip3 install pytest behave==1.2.6 - if: matrix.ubuntu == 18 + run: pip3 install pylint pytest behave==1.2.6 + if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }} + + - name: Install test prerequsites + run: sudo apt-get install -y -qq python3-pytest + if: matrix.ubuntu == 22 - name: Install latest pylint run: pip3 install pylint @@ -102,7 +112,7 @@ jobs: - name: PHP unit tests run: phpunit ./ working-directory: Nominatim/test/php - if: matrix.ubuntu == 20 + if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }} - name: Python unit tests run: $PYTEST test/python @@ -161,7 +171,7 @@ jobs: strategy: matrix: - name: [Ubuntu-18, Ubuntu-20] + name: [Ubuntu-18, Ubuntu-20, Ubuntu-22] include: - name: Ubuntu-18 flavour: ubuntu @@ -173,6 +183,11 @@ jobs: image: "ubuntu:20.04" ubuntu: 20 install_mode: install-apache + - name: Ubuntu-22 + flavour: ubuntu + image: "ubuntu:22.04" + ubuntu: 22 + install_mode: install-apache container: image: ${{ matrix.image }} diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 3b9a316b..0ccc5974 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -26,6 +26,7 @@ ADD_CUSTOM_TARGET(doc COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Centos-8.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Centos-8.md COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md COMMAND PYTHONPATH=${PROJECT_SOURCE_DIR} mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml ) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index a156fb71..48fe1d0d 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -46,6 +46,7 @@ nav: - 'Installation on CentOS 8' : 'appendix/Install-on-Centos-8.md' - 'Installation on Ubuntu 18' : 'appendix/Install-on-Ubuntu-18.md' - 'Installation on Ubuntu 20' : 'appendix/Install-on-Ubuntu-20.md' + - 'Installation on Ubuntu 22' : 'appendix/Install-on-Ubuntu-22.md' markdown_extensions: - codehilite - admonition diff --git a/lib-php/ParameterParser.php b/lib-php/ParameterParser.php index 98b95388..070be36c 100644 --- a/lib-php/ParameterParser.php +++ b/lib-php/ParameterParser.php @@ -70,7 +70,7 @@ class ParameterParser return $sDefault; } - if (!in_array($this->aParams[$sName], $aValues)) { + if (!in_array($this->aParams[$sName], $aValues, true)) { userError("Parameter '$sName' must be one of: ".join(', ', $aValues)); } diff --git a/lib-php/PlaceLookup.php b/lib-php/PlaceLookup.php index 715f1ced..33156537 100644 --- a/lib-php/PlaceLookup.php +++ b/lib-php/PlaceLookup.php @@ -445,7 +445,7 @@ class PlaceLookup if ($this->bExtraTags) { if ($aPlace['extra']) { - $aPlace['sExtraTags'] = json_decode($aPlace['extra']); + $aPlace['sExtraTags'] = json_decode($aPlace['extra'], true); } else { $aPlace['sExtraTags'] = (object) array(); } @@ -482,7 +482,7 @@ class PlaceLookup return (object) array(); } - $aFullNames = json_decode($sNames); + $aFullNames = json_decode($sNames, true); $aNames = array(); foreach ($aFullNames as $sKey => $sValue) { diff --git a/lib-php/SimpleWordList.php b/lib-php/SimpleWordList.php index ecd02153..7009d370 100644 --- a/lib-php/SimpleWordList.php +++ b/lib-php/SimpleWordList.php @@ -120,13 +120,18 @@ class SimpleWordList return array_slice($aWordSets, 0, SimpleWordList::MAX_WORDSETS); } + /** + * Custom search routine which takes two arrays. The array with the fewest + * items wins. If same number of items then the one with the longest first + * element wins. + */ public static function cmpByArraylen($aA, $aB) { $iALen = count($aA); $iBLen = count($aB); if ($iALen == $iBLen) { - return 0; + return strlen($aB[0]) <=> strlen($aA[0]); } return ($iALen < $iBLen) ? -1 : 1; diff --git a/lib-php/template/address-geocodejson.php b/lib-php/template/address-geocodejson.php index 8a0a6289..d54aef40 100644 --- a/lib-php/template/address-geocodejson.php +++ b/lib-php/template/address-geocodejson.php @@ -56,7 +56,7 @@ if (empty($aPlace)) { } if (isset($aPlace['asgeojson'])) { - $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson']); + $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson'], true); } else { $aFilteredPlaces['geometry'] = array( 'type' => 'Point', diff --git a/lib-php/template/address-geojson.php b/lib-php/template/address-geojson.php index 206b959f..dc3c3832 100644 --- a/lib-php/template/address-geojson.php +++ b/lib-php/template/address-geojson.php @@ -65,7 +65,7 @@ if (empty($aPlace)) { } if (isset($aPlace['asgeojson'])) { - $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson']); + $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson'], true); } else { $aFilteredPlaces['geometry'] = array( 'type' => 'Point', diff --git a/lib-php/template/address-json.php b/lib-php/template/address-json.php index 1a429acb..0766eaf4 100644 --- a/lib-php/template/address-json.php +++ b/lib-php/template/address-json.php @@ -63,7 +63,7 @@ if (empty($aPlace)) { } if (isset($aPlace['asgeojson'])) { - $aFilteredPlaces['geojson'] = json_decode($aPlace['asgeojson']); + $aFilteredPlaces['geojson'] = json_decode($aPlace['asgeojson'], true); } if (isset($aPlace['assvg'])) { diff --git a/lib-php/template/details-json.php b/lib-php/template/details-json.php index 68a0ce25..ae80a85b 100644 --- a/lib-php/template/details-json.php +++ b/lib-php/template/details-json.php @@ -48,7 +48,7 @@ $aPlaceDetails['centroid'] = array( 'coordinates' => array( (float) $aPointDetails['lon'], (float) $aPointDetails['lat'] ) ); -$aPlaceDetails['geometry'] = json_decode($aPointDetails['asgeojson']); +$aPlaceDetails['geometry'] = json_decode($aPointDetails['asgeojson'], true); $funcMapAddressLine = function ($aFull) { return array( diff --git a/lib-php/template/search-batch-json.php b/lib-php/template/search-batch-json.php index cbf25c87..430237a2 100644 --- a/lib-php/template/search-batch-json.php +++ b/lib-php/template/search-batch-json.php @@ -60,7 +60,7 @@ foreach ($aBatchResults as $aSearchResults) { } if (isset($aPointDetails['asgeojson'])) { - $aPlace['geojson'] = json_decode($aPointDetails['asgeojson']); + $aPlace['geojson'] = json_decode($aPointDetails['asgeojson'], true); } if (isset($aPointDetails['assvg'])) { diff --git a/lib-php/template/search-geocodejson.php b/lib-php/template/search-geocodejson.php index 5439e3cf..bba41a0d 100644 --- a/lib-php/template/search-geocodejson.php +++ b/lib-php/template/search-geocodejson.php @@ -46,7 +46,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) { } if (isset($aPointDetails['asgeojson'])) { - $aPlace['geometry'] = json_decode($aPointDetails['asgeojson']); + $aPlace['geometry'] = json_decode($aPointDetails['asgeojson'], true); } else { $aPlace['geometry'] = array( 'type' => 'Point', diff --git a/lib-php/template/search-geojson.php b/lib-php/template/search-geojson.php index f517142f..7665700d 100644 --- a/lib-php/template/search-geojson.php +++ b/lib-php/template/search-geojson.php @@ -54,7 +54,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) { } if (isset($aPointDetails['asgeojson'])) { - $aPlace['geometry'] = json_decode($aPointDetails['asgeojson']); + $aPlace['geometry'] = json_decode($aPointDetails['asgeojson'], true); } else { $aPlace['geometry'] = array( 'type' => 'Point', diff --git a/lib-php/template/search-json.php b/lib-php/template/search-json.php index 3f9e0a57..5fb13020 100644 --- a/lib-php/template/search-json.php +++ b/lib-php/template/search-json.php @@ -53,7 +53,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) { } if (isset($aPointDetails['asgeojson'])) { - $aPlace['geojson'] = json_decode($aPointDetails['asgeojson']); + $aPlace['geojson'] = json_decode($aPointDetails['asgeojson'], true); } if (isset($aPointDetails['assvg'])) { diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index f0ec358b..73095468 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -63,8 +63,9 @@ class SetupAll: @staticmethod - def run(args): - from ..tools import database_import, refresh, postcodes, freeze, country_info + def run(args): # pylint: disable=too-many-statements + from ..data import country_info + from ..tools import database_import, refresh, postcodes, freeze from ..indexer.indexer import Indexer country_info.setup_country_config(args.config) diff --git a/nominatim/tools/country_info.py b/nominatim/data/country_info.py similarity index 100% rename from nominatim/tools/country_info.py rename to nominatim/data/country_info.py diff --git a/nominatim/indexer/place_info.py b/nominatim/data/place_info.py similarity index 87% rename from nominatim/indexer/place_info.py rename to nominatim/data/place_info.py index 87ecb731..d2ba3979 100644 --- a/nominatim/indexer/place_info.py +++ b/nominatim/data/place_info.py @@ -9,8 +9,6 @@ Wrapper around place information the indexer gets from the database and hands to the tokenizer. """ -import psycopg2.extras - class PlaceInfo: """ Data class containing all information the tokenizer gets about a place it should process the names for. @@ -20,13 +18,6 @@ class PlaceInfo: self._info = info - def analyze(self, analyzer): - """ Process this place with the given tokenizer and return the - result in psycopg2-compatible Json. - """ - return psycopg2.extras.Json(analyzer.process_place(self)) - - @property def name(self): """ A dictionary with the names of the place or None if the place diff --git a/nominatim/data/postcode_format.py b/nominatim/data/postcode_format.py index 6ae43b7d..366ea505 100644 --- a/nominatim/data/postcode_format.py +++ b/nominatim/data/postcode_format.py @@ -11,7 +11,7 @@ format. import re from nominatim.errors import UsageError -from nominatim.tools import country_info +from nominatim.data import country_info class CountryPostcodeMatcher: """ Matches and formats a postcode according to a format definition diff --git a/nominatim/indexer/runners.py b/nominatim/indexer/runners.py index 9a30ffe6..c8495ee4 100644 --- a/nominatim/indexer/runners.py +++ b/nominatim/indexer/runners.py @@ -11,14 +11,17 @@ tasks. import functools from psycopg2 import sql as pysql +import psycopg2.extras -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo # pylint: disable=C0111 def _mk_valuelist(template, num): return pysql.SQL(',').join([pysql.SQL(template)] * num) +def _analyze_place(place, analyzer): + return psycopg2.extras.Json(analyzer.process_place(PlaceInfo(place))) class AbstractPlacexRunner: """ Returns SQL commands for indexing of the placex table. @@ -56,7 +59,7 @@ class AbstractPlacexRunner: for place in places: for field in ('place_id', 'name', 'address', 'linked_place_id'): values.append(place[field]) - values.append(PlaceInfo(place).analyze(self.analyzer)) + values.append(_analyze_place(place, self.analyzer)) worker.perform(self._index_sql(len(places)), values) @@ -150,7 +153,7 @@ class InterpolationRunner: values = [] for place in places: values.extend((place[x] for x in ('place_id', 'address'))) - values.append(PlaceInfo(place).analyze(self.analyzer)) + values.append(_analyze_place(place, self.analyzer)) worker.perform(self._index_sql(len(places)), values) diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py index f81b3bc2..70a54bfd 100644 --- a/nominatim/tokenizer/base.py +++ b/nominatim/tokenizer/base.py @@ -12,7 +12,7 @@ from abc import ABC, abstractmethod from typing import List, Tuple, Dict, Any from nominatim.config import Configuration -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo # pylint: disable=unnecessary-pass diff --git a/nominatim/tokenizer/icu_rule_loader.py b/nominatim/tokenizer/icu_rule_loader.py index 8a564355..035b6698 100644 --- a/nominatim/tokenizer/icu_rule_loader.py +++ b/nominatim/tokenizer/icu_rule_loader.py @@ -17,7 +17,7 @@ from nominatim.db.properties import set_property, get_property from nominatim.errors import UsageError from nominatim.tokenizer.place_sanitizer import PlaceSanitizer from nominatim.tokenizer.icu_token_analysis import ICUTokenAnalysis -import nominatim.tools.country_info +import nominatim.data.country_info LOG = logging.getLogger() @@ -46,7 +46,7 @@ class ICURuleLoader: config='TOKENIZER_CONFIG') # Make sure country information is available to analyzers and sanitizers. - nominatim.tools.country_info.setup_country_config(config) + nominatim.data.country_info.setup_country_config(config) self.normalization_rules = self._cfg_to_icu_rules(rules, 'normalization') self.transliteration_rules = self._cfg_to_icu_rules(rules, 'transliteration') diff --git a/nominatim/tokenizer/icu_tokenizer.py b/nominatim/tokenizer/icu_tokenizer.py index a6ff08a4..171d4392 100644 --- a/nominatim/tokenizer/icu_tokenizer.py +++ b/nominatim/tokenizer/icu_tokenizer.py @@ -16,7 +16,7 @@ from textwrap import dedent from nominatim.db.connection import connect from nominatim.db.utils import CopyBuffer from nominatim.db.sql_preprocessor import SQLPreprocessor -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo from nominatim.tokenizer.icu_rule_loader import ICURuleLoader from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer diff --git a/nominatim/tokenizer/sanitizers/tag_analyzer_by_language.py b/nominatim/tokenizer/sanitizers/tag_analyzer_by_language.py index 9a99d127..d3413c1a 100644 --- a/nominatim/tokenizer/sanitizers/tag_analyzer_by_language.py +++ b/nominatim/tokenizer/sanitizers/tag_analyzer_by_language.py @@ -30,7 +30,7 @@ Arguments: any analyzer tagged) is retained. (default: replace) """ -from nominatim.tools import country_info +from nominatim.data import country_info class _AnalyzerByLanguage: """ Processor for tagging the language of names in a place. diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py index 6e37df5e..e78dcd8f 100644 --- a/nominatim/tools/tiger_data.py +++ b/nominatim/tools/tiger_data.py @@ -13,11 +13,13 @@ import logging import os import tarfile +from psycopg2.extras import Json + from nominatim.db.connection import connect from nominatim.db.async_connection import WorkerPool from nominatim.db.sql_preprocessor import SQLPreprocessor from nominatim.errors import UsageError -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo LOG = logging.getLogger() @@ -87,7 +89,7 @@ def handle_threaded_sql_statements(pool, fd, analyzer): address = dict(street=row['street'], postcode=row['postcode']) args = ('SRID=4326;' + row['geometry'], int(row['from']), int(row['to']), row['interpolation'], - PlaceInfo({'address': address}).analyze(analyzer), + Json(analyzer.process_place(PlaceInfo({'address': address}))), analyzer.normalize_postcode(row['postcode'])) except ValueError: continue diff --git a/test/php/Nominatim/ParameterParserTest.php b/test/php/Nominatim/ParameterParserTest.php index 1488c987..7381bdf8 100644 --- a/test/php/Nominatim/ParameterParserTest.php +++ b/test/php/Nominatim/ParameterParserTest.php @@ -137,6 +137,9 @@ class ParameterParserTest extends \PHPUnit\Framework\TestCase public function testGetSet() { + $this->expectException(\Exception::class); + $this->expectExceptionMessage("Parameter 'val3' must be one of: foo, bar"); + $oParams = new ParameterParser(array( 'val1' => 'foo', 'val2' => '', @@ -148,7 +151,7 @@ class ParameterParserTest extends \PHPUnit\Framework\TestCase $this->assertSame('foo', $oParams->getSet('val1', array('foo', 'bar'))); $this->assertSame(false, $oParams->getSet('val2', array('foo', 'bar'))); - $this->assertSame(0, $oParams->getSet('val3', array('foo', 'bar'))); + $oParams->getSet('val3', array('foo', 'bar')); } diff --git a/test/php/Nominatim/SimpleWordListTest.php b/test/php/Nominatim/SimpleWordListTest.php index fe543b3f..69cb5180 100644 --- a/test/php/Nominatim/SimpleWordListTest.php +++ b/test/php/Nominatim/SimpleWordListTest.php @@ -77,7 +77,7 @@ class SimpleWordListTest extends \PHPUnit\Framework\TestCase $oList = new SimpleWordList('a b c'); $this->assertEquals( - '(a b c),(a|b c),(a b|c),(a|b|c)', + '(a b c),(a b|c),(a|b c),(a|b|c)', $this->serializeSets($oList->getWordSets(new TokensFullSet())) ); @@ -88,6 +88,22 @@ class SimpleWordListTest extends \PHPUnit\Framework\TestCase ); } + public function testCmpByArraylen() + { + // Array elements are phrases, we want to sort so longest phrases are first + $aList1 = array('hackney', 'bridge', 'london', 'england'); + $aList2 = array('hackney', 'london', 'bridge'); + $aList3 = array('bridge', 'hackney', 'london', 'england'); + + $this->assertEquals(0, \Nominatim\SimpleWordList::cmpByArraylen($aList1, $aList1)); + + // list2 "wins". Less array elements + $this->assertEquals(1, \Nominatim\SimpleWordList::cmpByArraylen($aList1, $aList2)); + $this->assertEquals(-1, \Nominatim\SimpleWordList::cmpByArraylen($aList2, $aList3)); + + // list1 "wins". Same number of array elements but longer first element + $this->assertEquals(-1, \Nominatim\SimpleWordList::cmpByArraylen($aList1, $aList3)); + } public function testMaxWordSets() { diff --git a/test/python/cli/test_cmd_import.py b/test/python/cli/test_cmd_import.py index 84c7743a..d545c760 100644 --- a/test/python/cli/test_cmd_import.py +++ b/test/python/cli/test_cmd_import.py @@ -10,7 +10,7 @@ Tests for import command of the command-line interface wrapper. import pytest import nominatim.tools.database_import -import nominatim.tools.country_info +import nominatim.data.country_info import nominatim.tools.refresh import nominatim.tools.postcodes import nominatim.indexer.indexer @@ -37,7 +37,7 @@ class TestCliImportWithDb: def test_import_full(self, mock_func_factory, with_updates, place_table, property_table): mocks = [ mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'), - mock_func_factory(nominatim.tools.country_info, 'setup_country_tables'), + mock_func_factory(nominatim.data.country_info, 'setup_country_tables'), mock_func_factory(nominatim.tools.database_import, 'import_osm_data'), mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'), mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'), @@ -46,7 +46,7 @@ class TestCliImportWithDb: mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'), mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'), mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), - mock_func_factory(nominatim.tools.country_info, 'create_country_names'), + mock_func_factory(nominatim.data.country_info, 'create_country_names'), mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_config'), mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'), mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'), @@ -76,7 +76,7 @@ class TestCliImportWithDb: mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'), mock_func_factory(nominatim.tools.database_import, 'load_data'), mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), - mock_func_factory(nominatim.tools.country_info, 'create_country_names'), + mock_func_factory(nominatim.data.country_info, 'create_country_names'), mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'), mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim.tools.refresh, 'setup_website'), @@ -94,7 +94,7 @@ class TestCliImportWithDb: temp_db_conn): mocks = [ mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), - mock_func_factory(nominatim.tools.country_info, 'create_country_names'), + mock_func_factory(nominatim.data.country_info, 'create_country_names'), mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim.tools.refresh, 'setup_website'), mock_func_factory(nominatim.db.properties, 'set_property') @@ -115,7 +115,7 @@ class TestCliImportWithDb: def test_import_continue_postprocess(self, mock_func_factory): mocks = [ mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), - mock_func_factory(nominatim.tools.country_info, 'create_country_names'), + mock_func_factory(nominatim.data.country_info, 'create_country_names'), mock_func_factory(nominatim.tools.refresh, 'setup_website'), mock_func_factory(nominatim.db.properties, 'set_property') ] diff --git a/test/python/tools/test_country_info.py b/test/python/data/test_country_info.py similarity index 99% rename from test/python/tools/test_country_info.py rename to test/python/data/test_country_info.py index 3f00d54e..2234f40d 100644 --- a/test/python/tools/test_country_info.py +++ b/test/python/data/test_country_info.py @@ -10,7 +10,7 @@ Tests for function that handle country properties. from textwrap import dedent import pytest -from nominatim.tools import country_info +from nominatim.data import country_info @pytest.fixture def loaded_country(def_config): diff --git a/test/python/db/test_utils.py b/test/python/db/test_utils.py index 3991dff3..e0be962f 100644 --- a/test/python/db/test_utils.py +++ b/test/python/db/test_utils.py @@ -65,7 +65,7 @@ class TestCopyBuffer: @pytest.fixture(autouse=True) def setup_test_table(self, table_factory): - table_factory(self.TABLE_NAME, 'colA INT, colB TEXT') + table_factory(self.TABLE_NAME, 'col_a INT, col_b TEXT') def table_rows(self, cursor): @@ -92,7 +92,7 @@ class TestCopyBuffer: buf.add('foo') buf.copy_out(temp_db_cursor, self.TABLE_NAME, - columns=['colB']) + columns=['col_b']) assert self.table_rows(temp_db_cursor) == {(None, 'foo')} @@ -103,7 +103,7 @@ class TestCopyBuffer: buf.add(' two ', 2) buf.copy_out(temp_db_cursor, self.TABLE_NAME, - columns=['colB', 'colA']) + columns=['col_b', 'col_a']) assert self.table_rows(temp_db_cursor) == {(1, 'one'), (2, ' two ')} @@ -115,7 +115,7 @@ class TestCopyBuffer: buf.add('\\N') buf.copy_out(temp_db_cursor, self.TABLE_NAME, - columns=['colB']) + columns=['col_b']) assert self.table_rows(temp_db_cursor) == {(None, 'foo\tbar'), (None, 'sun\nson'), @@ -128,7 +128,7 @@ class TestCopyBufferJson: @pytest.fixture(autouse=True) def setup_test_table(self, table_factory): - table_factory(self.TABLE_NAME, 'colA INT, colB JSONB') + table_factory(self.TABLE_NAME, 'col_a INT, col_b JSONB') def table_rows(self, cursor): diff --git a/test/python/dummy_tokenizer.py b/test/python/dummy_tokenizer.py index 57bf2f2f..ed32c650 100644 --- a/test/python/dummy_tokenizer.py +++ b/test/python/dummy_tokenizer.py @@ -7,7 +7,7 @@ """ Tokenizer for testing. """ -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo from nominatim.config import Configuration def create(dsn, data_dir): diff --git a/test/python/tokenizer/sanitizers/test_clean_housenumbers.py b/test/python/tokenizer/sanitizers/test_clean_housenumbers.py index 34cc7413..128e1201 100644 --- a/test/python/tokenizer/sanitizers/test_clean_housenumbers.py +++ b/test/python/tokenizer/sanitizers/test_clean_housenumbers.py @@ -10,7 +10,7 @@ Tests for the sanitizer that normalizes housenumbers. import pytest from nominatim.tokenizer.place_sanitizer import PlaceSanitizer -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo @pytest.fixture def sanitize(request): diff --git a/test/python/tokenizer/sanitizers/test_clean_postcodes.py b/test/python/tokenizer/sanitizers/test_clean_postcodes.py index 44376196..237527f1 100644 --- a/test/python/tokenizer/sanitizers/test_clean_postcodes.py +++ b/test/python/tokenizer/sanitizers/test_clean_postcodes.py @@ -10,8 +10,8 @@ Tests for the sanitizer that normalizes postcodes. import pytest from nominatim.tokenizer.place_sanitizer import PlaceSanitizer -from nominatim.indexer.place_info import PlaceInfo -from nominatim.tools import country_info +from nominatim.data.place_info import PlaceInfo +from nominatim.data import country_info @pytest.fixture def sanitize(def_config, request): diff --git a/test/python/tokenizer/sanitizers/test_split_name_list.py b/test/python/tokenizer/sanitizers/test_split_name_list.py index 47bd1e44..67157fba 100644 --- a/test/python/tokenizer/sanitizers/test_split_name_list.py +++ b/test/python/tokenizer/sanitizers/test_split_name_list.py @@ -10,7 +10,7 @@ Tests for the sanitizer that splits multivalue lists. import pytest from nominatim.tokenizer.place_sanitizer import PlaceSanitizer -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo from nominatim.errors import UsageError diff --git a/test/python/tokenizer/sanitizers/test_strip_brace_terms.py b/test/python/tokenizer/sanitizers/test_strip_brace_terms.py index c37562dd..eb554364 100644 --- a/test/python/tokenizer/sanitizers/test_strip_brace_terms.py +++ b/test/python/tokenizer/sanitizers/test_strip_brace_terms.py @@ -10,7 +10,7 @@ Tests for the sanitizer that handles braced suffixes. import pytest from nominatim.tokenizer.place_sanitizer import PlaceSanitizer -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo def run_sanitizer_on(**kwargs): place = PlaceInfo({'name': kwargs}) diff --git a/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py b/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py index dfd25113..306b8027 100644 --- a/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py +++ b/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py @@ -9,9 +9,9 @@ Tests for the sanitizer that enables language-dependent analyzers. """ import pytest -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo from nominatim.tokenizer.place_sanitizer import PlaceSanitizer -from nominatim.tools.country_info import setup_country_config +from nominatim.data.country_info import setup_country_config class TestWithDefaults: diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py index b9de97bc..7f0ffce1 100644 --- a/test/python/tokenizer/test_icu.py +++ b/test/python/tokenizer/test_icu.py @@ -17,7 +17,7 @@ from nominatim.tokenizer import icu_tokenizer import nominatim.tokenizer.icu_rule_loader from nominatim.db import properties from nominatim.db.sql_preprocessor import SQLPreprocessor -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo from mock_icu_word_table import MockIcuWordTable diff --git a/test/python/tokenizer/test_legacy.py b/test/python/tokenizer/test_legacy.py index 8f79e242..57a82b8a 100644 --- a/test/python/tokenizer/test_legacy.py +++ b/test/python/tokenizer/test_legacy.py @@ -12,7 +12,7 @@ import re import pytest -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo from nominatim.tokenizer import legacy_tokenizer from nominatim.db import properties from nominatim.errors import UsageError diff --git a/test/python/tokenizer/test_place_sanitizer.py b/test/python/tokenizer/test_place_sanitizer.py index 2dd4e58a..31401bd1 100644 --- a/test/python/tokenizer/test_place_sanitizer.py +++ b/test/python/tokenizer/test_place_sanitizer.py @@ -11,7 +11,7 @@ import pytest from nominatim.errors import UsageError import nominatim.tokenizer.place_sanitizer as sanitizer -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo def test_placeinfo_clone_new_name(): diff --git a/test/python/tools/test_exec_utils.py b/test/python/tools/test_exec_utils.py index 78650180..26ea92b2 100644 --- a/test/python/tools/test_exec_utils.py +++ b/test/python/tools/test_exec_utils.py @@ -122,7 +122,13 @@ class TestRunApiScript: @staticmethod def test_fail_on_error_output(tmp_path): - (tmp_path / 'website' / 'bad.php').write_text(" + Options FollowSymLinks MultiViews + AddType text/html .php + DirectoryIndex search.php + Require all granted + + +Alias /nominatim $USERHOME/nominatim-project/website +EOFAPACHECONF +#DOCS:``` + +# +# Then enable the configuration and restart apache +# + + sudo a2enconf nominatim +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo apache2ctl start #DOCS: +else #DOCS: + sudo systemctl restart apache2 +fi #DOCS: + +# The Nominatim API is now available at `http://localhost/nominatim/`. + +fi #DOCS: + +# +# Option 2: Using nginx +# --------------------- +# +if [ "x$2" == "xinstall-nginx" ]; then #DOCS: + +# Nginx has no native support for php scripts. You need to set up php-fpm for +# this purpose. First install nginx and php-fpm: + + sudo apt install -y nginx php-fpm + +# You need to configure php-fpm to listen on a Unix socket. + +#DOCS:```sh +sudo tee /etc/php/8.1/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF +[www] +; Replace the tcp listener and add the unix socket +listen = /var/run/php8.1-fpm.sock + +; Ensure that the daemon runs as the correct user +listen.owner = www-data +listen.group = www-data +listen.mode = 0666 + +; Unix user of FPM processes +user = www-data +group = www-data + +; Choose process manager type (static, dynamic, ondemand) +pm = ondemand +pm.max_children = 5 +EOF_PHP_FPM_CONF +#DOCS:``` + +# Then create a Nginx configuration to forward http requests to that socket. + +#DOCS:```sh +sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF +server { + listen 80 default_server; + listen [::]:80 default_server; + + root $USERHOME/nominatim-project/website; + index search.php index.html; + location / { + try_files \$uri \$uri/ @php; + } + + location @php { + fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php"; + fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php"; + fastcgi_param QUERY_STRING \$args; + fastcgi_pass unix:/var/run/php8.1-fpm.sock; + fastcgi_index index.php; + include fastcgi_params; + } + + location ~ [^/]\.php(/|$) { + fastcgi_split_path_info ^(.+?\.php)(/.*)$; + if (!-f \$document_root\$fastcgi_script_name) { + return 404; + } + fastcgi_pass unix:/var/run/php7.4-fpm.sock; + fastcgi_index search.php; + include fastcgi.conf; + } +} +EOF_NGINX_CONF +#DOCS:``` + +# If you have some errors, make sure that php8.1-fpm.sock is well under +# /var/run/ and not under /var/run/php. Otherwise change the Nginx configuration +# to /var/run/php/php8.1-fpm.sock. +# +# Enable the configuration and restart Nginx +# + +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo /usr/sbin/php-fpm8.1 --nodaemonize --fpm-config /etc/php/8.1/fpm/php-fpm.conf & #DOCS: + sudo /usr/sbin/nginx & #DOCS: +else #DOCS: + sudo systemctl restart php8.1-fpm nginx +fi #DOCS: + +# The Nominatim API is now available at `http://localhost/`. + + + +fi #DOCS: