X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/44cfce1ca4caf9fb716ff7c37682427d9bd3d31e..690f9ec06fd7a08b41de027427d8a1d3d0e3ff2e:/test/python/tokenizer/test_icu.py diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py index 22112220..372df9d2 100644 --- a/test/python/tokenizer/test_icu.py +++ b/test/python/tokenizer/test_icu.py @@ -1,8 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. """ Tests for ICU tokenizer. """ import shutil import yaml +import itertools import pytest @@ -394,7 +401,9 @@ class TestPlaceAddress: @pytest.fixture(autouse=True) def setup(self, analyzer, sql_functions): - with analyzer(trans=(":: upper()", "'🜵' > ' '")) as anl: + hnr = {'step': 'clean-housenumbers', + 'filter-kind': ['housenumber', 'conscriptionnumber', 'streetnumber']} + with analyzer(trans=(":: upper()", "'🜵' > ' '"), sanitizers=[hnr]) as anl: self.analyzer = anl yield anl @@ -440,13 +449,6 @@ class TestPlaceAddress: assert info['hnr_tokens'] == "{-1}" - def test_process_place_housenumbers_lists(self, getorcreate_hnr_id): - info = self.process_address(conscriptionnumber='1; 2;3') - - assert set(info['hnr'].split(';')) == set(('1', '2', '3')) - assert info['hnr_tokens'] == "{-1,-2,-3}" - - def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id): info = self.process_address(housenumber='134', conscriptionnumber='134', @@ -514,6 +516,12 @@ class TestPlaceAddress: assert eval(info['place']) == self.name_token_set('HONU', 'LULU') + def test_process_place_place_extra(self): + info = self.process_address(**{'place:en': 'Honu Lulu'}) + + assert 'place' not in info + + def test_process_place_place_empty(self): info = self.process_address(place='🜵') @@ -533,9 +541,83 @@ class TestPlaceAddress: assert result == {'city': city, 'suburb': city, 'state': state} + def test_process_place_multiple_address_terms(self): + info = self.process_address(**{'city': 'Bruxelles', 'city:de': 'Brüssel'}) + + result = {k: eval(v) for k,v in info['addr'].items()} + + assert result == {'city': self.name_token_set('Bruxelles')} + + def test_process_place_address_terms_empty(self): info = self.process_address(country='de', city=' ', street='Hauptstr', full='right behind the church') assert 'addr' not in info + +class TestUpdateWordTokens: + + @pytest.fixture(autouse=True) + def setup(self, tokenizer_factory, table_factory, placex_table, word_table): + table_factory('search_name', 'place_id BIGINT, name_vector INT[]') + self.tok = tokenizer_factory() + + + @pytest.fixture + def search_entry(self, temp_db_cursor): + place_id = itertools.count(1000) + + def _insert(*args): + temp_db_cursor.execute("INSERT INTO search_name VALUES (%s, %s)", + (next(place_id), list(args))) + + return _insert + + + @pytest.mark.parametrize('hnr', ('1a', '1234567', '34 5')) + def test_remove_unused_housenumbers(self, word_table, hnr): + word_table.add_housenumber(1000, hnr) + + assert word_table.count_housenumbers() == 1 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 0 + + + def test_keep_unused_numeral_housenumbers(self, word_table): + word_table.add_housenumber(1000, '5432') + + assert word_table.count_housenumbers() == 1 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1 + + + def test_keep_housenumbers_from_search_name_table(self, word_table, search_entry): + word_table.add_housenumber(9999, '5432a') + word_table.add_housenumber(9991, '9 a') + search_entry(123, 9999, 34) + + assert word_table.count_housenumbers() == 2 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1 + + + def test_keep_housenumbers_from_placex_table(self, word_table, placex_table): + word_table.add_housenumber(9999, '5432a') + word_table.add_housenumber(9990, '34z') + placex_table.add(housenumber='34z') + placex_table.add(housenumber='25432a') + + assert word_table.count_housenumbers() == 2 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1 + + + def test_keep_housenumbers_from_placex_table_hnr_list(self, word_table, placex_table): + word_table.add_housenumber(9991, '9 b') + word_table.add_housenumber(9990, '34z') + placex_table.add(housenumber='9 a;9 b;9 c') + + assert word_table.count_housenumbers() == 2 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1