X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/b90e719da595b6760b39b7cd64ee29447de2d5e8..23eed4ff2fe4a987ff82d816fdedb9f2bbaa5d88:/test/python/tokenizer/test_icu.py diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py index 5dbe292e..2d9da69a 100644 --- a/test/python/tokenizer/test_icu.py +++ b/test/python/tokenizer/test_icu.py @@ -1,8 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. """ -Tests for Legacy ICU tokenizer. +Tests for ICU tokenizer. """ import shutil import yaml +import itertools import pytest @@ -10,7 +17,7 @@ from nominatim.tokenizer import icu_tokenizer import nominatim.tokenizer.icu_rule_loader from nominatim.db import properties from nominatim.db.sql_preprocessor import SQLPreprocessor -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo from mock_icu_word_table import MockIcuWordTable @@ -20,20 +27,17 @@ def word_table(temp_db_conn): @pytest.fixture -def test_config(def_config, tmp_path): - def_config.project_dir = tmp_path / 'project' - def_config.project_dir.mkdir() - +def test_config(project_env, tmp_path): sqldir = tmp_path / 'sql' sqldir.mkdir() (sqldir / 'tokenizer').mkdir() (sqldir / 'tokenizer' / 'icu_tokenizer.sql').write_text("SELECT 'a'") - shutil.copy(str(def_config.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'), + shutil.copy(str(project_env.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'), str(sqldir / 'tokenizer' / 'icu_tokenizer_tables.sql')) - def_config.lib_dir.sql = sqldir + project_env.lib_dir.sql = sqldir - return def_config + return project_env @pytest.fixture @@ -68,12 +72,19 @@ def analyzer(tokenizer_factory, test_config, monkeypatch, def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',), variants=('~gasse -> gasse', 'street => st', ), - sanitizers=[]): + sanitizers=[], with_housenumber=False, + with_postcode=False): cfgstr = {'normalization': list(norm), 'sanitizers': sanitizers, 'transliteration': list(trans), 'token-analysis': [{'analyzer': 'generic', 'variants': [{'words': list(variants)}]}]} + if with_housenumber: + cfgstr['token-analysis'].append({'id': '@housenumber', + 'analyzer': 'housenumbers'}) + if with_postcode: + cfgstr['token-analysis'].append({'id': '@postcode', + 'analyzer': 'postcodes'}) (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr)) tok.loader = nominatim.tokenizer.icu_rule_loader.ICURuleLoader(test_config) @@ -144,12 +155,6 @@ LANGUAGE plpgsql; """) -@pytest.fixture -def getorcreate_hnr_id(temp_db_cursor): - temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT) - RETURNS INTEGER AS $$ - SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""") - def test_init_new(tokenizer_factory, test_config, db_prop): tok = tokenizer_factory() @@ -197,6 +202,47 @@ def test_update_sql_functions(db_prop, temp_db_cursor, assert test_content == set((('1133', ), )) +def test_finalize_import(tokenizer_factory, temp_db_conn, + temp_db_cursor, test_config, sql_preprocessor_cfg): + func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql' + func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT + AS $$ SELECT 'b'::text $$ LANGUAGE SQL""") + + tok = tokenizer_factory() + tok.init_new_db(test_config) + + tok.finalize_import(test_config) + + temp_db_cursor.scalar('SELECT test()') == 'b' + + +def test_check_database(test_config, tokenizer_factory, + temp_db_cursor, sql_preprocessor_cfg): + tok = tokenizer_factory() + tok.init_new_db(test_config) + + assert tok.check_database(test_config) is None + + +def test_update_statistics_reverse_only(word_table, tokenizer_factory): + tok = tokenizer_factory() + tok.update_statistics() + + +def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory): + word_table.add_full_word(1000, 'hello') + table_factory('search_name', + 'place_id BIGINT, name_vector INT[]', + [(12, [1000])]) + tok = tokenizer_factory() + + tok.update_statistics() + + assert temp_db_cursor.scalar("""SELECT count(*) FROM word + WHERE type = 'W' and + (info->>'count')::int > 0""") > 0 + + def test_normalize_postcode(analyzer): with analyzer() as anl: anl.normalize_postcode('123') == '123' @@ -204,28 +250,69 @@ def test_normalize_postcode(analyzer): anl.normalize_postcode('38 Б') == '38 Б' -def test_update_postcodes_from_db_empty(analyzer, table_factory, word_table): - table_factory('location_postcode', 'postcode TEXT', - content=(('1234',), ('12 34',), ('AB23',), ('1234',))) +class TestPostcodes: - with analyzer() as anl: - anl.update_postcodes_from_db() + @pytest.fixture(autouse=True) + def setup(self, analyzer, sql_functions): + sanitizers = [{'step': 'clean-postcodes'}] + with analyzer(sanitizers=sanitizers, with_postcode=True) as anl: + self.analyzer = anl + yield anl - assert word_table.count() == 3 - assert word_table.get_postcodes() == {'1234', '12 34', 'AB23'} + def process_postcode(self, cc, postcode): + return self.analyzer.process_place(PlaceInfo({'country_code': cc, + 'address': {'postcode': postcode}})) -def test_update_postcodes_from_db_add_and_remove(analyzer, table_factory, word_table): - table_factory('location_postcode', 'postcode TEXT', - content=(('1234',), ('45BC', ), ('XX45', ))) - word_table.add_postcode(' 1234', '1234') - word_table.add_postcode(' 5678', '5678') - with analyzer() as anl: - anl.update_postcodes_from_db() + def test_update_postcodes_from_db_empty(self, table_factory, word_table): + table_factory('location_postcode', 'country_code TEXT, postcode TEXT', + content=(('de', '12345'), ('se', '132 34'), + ('bm', 'AB23'), ('fr', '12345'))) + + self.analyzer.update_postcodes_from_db() + + assert word_table.count() == 5 + assert word_table.get_postcodes() == {'12345', '132 34@132 34', 'AB 23@AB 23'} + + + def test_update_postcodes_from_db_ambigious(self, table_factory, word_table): + table_factory('location_postcode', 'country_code TEXT, postcode TEXT', + content=(('in', '123456'), ('sg', '123456'))) + + self.analyzer.update_postcodes_from_db() + + assert word_table.count() == 3 + assert word_table.get_postcodes() == {'123456', '123456@123 456'} + + + def test_update_postcodes_from_db_add_and_remove(self, table_factory, word_table): + table_factory('location_postcode', 'country_code TEXT, postcode TEXT', + content=(('ch', '1234'), ('bm', 'BC 45'), ('bm', 'XX45'))) + word_table.add_postcode(' 1234', '1234') + word_table.add_postcode(' 5678', '5678') + + self.analyzer.update_postcodes_from_db() + + assert word_table.count() == 5 + assert word_table.get_postcodes() == {'1234', 'BC 45@BC 45', 'XX 45@XX 45'} + + + def test_process_place_postcode_simple(self, word_table): + info = self.process_postcode('de', '12345') + + assert info['postcode'] == '12345' + + assert word_table.get_postcodes() == {'12345', } + + + def test_process_place_postcode_with_space(self, word_table): + info = self.process_postcode('in', '123 567') + + assert info['postcode'] == '123567' + + assert word_table.get_postcodes() == {'123567@123 567', } - assert word_table.count() == 3 - assert word_table.get_postcodes() == {'1234', '45BC', 'XX45'} def test_update_special_phrase_empty_table(analyzer, word_table): @@ -362,11 +449,20 @@ class TestPlaceAddress: @pytest.fixture(autouse=True) def setup(self, analyzer, sql_functions): - with analyzer(trans=(":: upper()", "'🜵' > ' '")) as anl: + hnr = {'step': 'clean-housenumbers', + 'filter-kind': ['housenumber', 'conscriptionnumber', 'streetnumber']} + with analyzer(trans=(":: upper()", "'🜵' > ' '"), sanitizers=[hnr]) as anl: self.analyzer = anl yield anl + @pytest.fixture + def getorcreate_hnr_id(self, temp_db_cursor): + temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT) + RETURNS INTEGER AS $$ + SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""") + + def process_address(self, **kwargs): return self.analyzer.process_place(PlaceInfo({'address': kwargs})) @@ -386,13 +482,6 @@ class TestPlaceAddress: assert word_table.get_postcodes() == {pcode, } - @pytest.mark.parametrize('pcode', ['12:23', 'ab;cd;f', '123;836']) - def test_process_place_bad_postcode(self, word_table, pcode): - self.process_address(postcode=pcode) - - assert not word_table.get_postcodes() - - @pytest.mark.parametrize('hnr', ['123a', '1', '101']) def test_process_place_housenumbers_simple(self, hnr, getorcreate_hnr_id): info = self.process_address(housenumber=hnr) @@ -401,13 +490,6 @@ class TestPlaceAddress: assert info['hnr_tokens'] == "{-1}" - def test_process_place_housenumbers_lists(self, getorcreate_hnr_id): - info = self.process_address(conscriptionnumber='1; 2;3') - - assert set(info['hnr'].split(';')) == set(('1', '2', '3')) - assert info['hnr_tokens'] == "{-1,-2,-3}" - - def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id): info = self.process_address(housenumber='134', conscriptionnumber='134', @@ -432,15 +514,41 @@ class TestPlaceAddress: def test_process_place_street(self): + self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road'}})) + info = self.process_address(street='Grand Road') + + assert eval(info['street']) == self.name_token_set('#Grand Road') + + + def test_process_place_nonexisting_street(self): info = self.process_address(street='Grand Road') - assert eval(info['street']) == self.name_token_set('GRAND', 'ROAD') + assert info['street'] == '{}' + + + def test_process_place_multiple_street_tags(self): + self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road', + 'ref': '05989'}})) + info = self.process_address(**{'street': 'Grand Road', + 'street:sym_ul': '05989'}) + + assert eval(info['street']) == self.name_token_set('#Grand Road', '#05989') def test_process_place_street_empty(self): info = self.process_address(street='🜵') - assert 'street' not in info + assert info['street'] == '{}' + + + def test_process_place_street_from_cache(self): + self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road'}})) + self.process_address(street='Grand Road') + + # request address again + info = self.process_address(street='Grand Road') + + assert eval(info['street']) == self.name_token_set('#Grand Road') def test_process_place_place(self): @@ -449,6 +557,12 @@ class TestPlaceAddress: assert eval(info['place']) == self.name_token_set('HONU', 'LULU') + def test_process_place_place_extra(self): + info = self.process_address(**{'place:en': 'Honu Lulu'}) + + assert 'place' not in info + + def test_process_place_place_empty(self): info = self.process_address(place='🜵') @@ -468,9 +582,156 @@ class TestPlaceAddress: assert result == {'city': city, 'suburb': city, 'state': state} + def test_process_place_multiple_address_terms(self): + info = self.process_address(**{'city': 'Bruxelles', 'city:de': 'Brüssel'}) + + result = {k: eval(v) for k,v in info['addr'].items()} + + assert result == {'city': self.name_token_set('Bruxelles')} + + def test_process_place_address_terms_empty(self): info = self.process_address(country='de', city=' ', street='Hauptstr', full='right behind the church') assert 'addr' not in info + +class TestPlaceHousenumberWithAnalyser: + + @pytest.fixture(autouse=True) + def setup(self, analyzer, sql_functions): + hnr = {'step': 'clean-housenumbers', + 'filter-kind': ['housenumber', 'conscriptionnumber', 'streetnumber']} + with analyzer(trans=(":: upper()", "'🜵' > ' '"), sanitizers=[hnr], with_housenumber=True) as anl: + self.analyzer = anl + yield anl + + + @pytest.fixture + def getorcreate_hnr_id(self, temp_db_cursor): + temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION create_analyzed_hnr_id(norm_term TEXT, lookup_terms TEXT[]) + RETURNS INTEGER AS $$ + SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""") + + + def process_address(self, **kwargs): + return self.analyzer.process_place(PlaceInfo({'address': kwargs})) + + + def name_token_set(self, *expected_terms): + tokens = self.analyzer.get_word_token_info(expected_terms) + for token in tokens: + assert token[2] is not None, "No token for {0}".format(token) + + return set((t[2] for t in tokens)) + + + @pytest.mark.parametrize('hnr', ['123 a', '1', '101']) + def test_process_place_housenumbers_simple(self, hnr, getorcreate_hnr_id): + info = self.process_address(housenumber=hnr) + + assert info['hnr'] == hnr.upper() + assert info['hnr_tokens'] == "{-1}" + + + def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id): + info = self.process_address(housenumber='134', + conscriptionnumber='134', + streetnumber='99a') + + assert set(info['hnr'].split(';')) == set(('134', '99 A')) + assert info['hnr_tokens'] == "{-1,-2}" + + + def test_process_place_housenumbers_cached(self, getorcreate_hnr_id): + info = self.process_address(housenumber="45") + assert info['hnr_tokens'] == "{-1}" + + info = self.process_address(housenumber="46") + assert info['hnr_tokens'] == "{-2}" + + info = self.process_address(housenumber="41;45") + assert eval(info['hnr_tokens']) == {-1, -3} + + info = self.process_address(housenumber="41") + assert eval(info['hnr_tokens']) == {-3} + + +class TestUpdateWordTokens: + + @pytest.fixture(autouse=True) + def setup(self, tokenizer_factory, table_factory, placex_table, word_table): + table_factory('search_name', 'place_id BIGINT, name_vector INT[]') + self.tok = tokenizer_factory() + + + @pytest.fixture + def search_entry(self, temp_db_cursor): + place_id = itertools.count(1000) + + def _insert(*args): + temp_db_cursor.execute("INSERT INTO search_name VALUES (%s, %s)", + (next(place_id), list(args))) + + return _insert + + + @pytest.fixture(params=['simple', 'analyzed']) + def add_housenumber(self, request, word_table): + if request.param == 'simple': + def _make(hid, hnr): + word_table.add_housenumber(hid, hnr) + elif request.param == 'analyzed': + def _make(hid, hnr): + word_table.add_housenumber(hid, [hnr]) + + return _make + + + @pytest.mark.parametrize('hnr', ('1a', '1234567', '34 5')) + def test_remove_unused_housenumbers(self, add_housenumber, word_table, hnr): + word_table.add_housenumber(1000, hnr) + + assert word_table.count_housenumbers() == 1 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 0 + + + def test_keep_unused_numeral_housenumbers(self, add_housenumber, word_table): + add_housenumber(1000, '5432') + + assert word_table.count_housenumbers() == 1 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1 + + + def test_keep_housenumbers_from_search_name_table(self, add_housenumber, word_table, search_entry): + add_housenumber(9999, '5432a') + add_housenumber(9991, '9 a') + search_entry(123, 9999, 34) + + assert word_table.count_housenumbers() == 2 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1 + + + def test_keep_housenumbers_from_placex_table(self, add_housenumber, word_table, placex_table): + add_housenumber(9999, '5432a') + add_housenumber(9990, '34z') + placex_table.add(housenumber='34z') + placex_table.add(housenumber='25432a') + + assert word_table.count_housenumbers() == 2 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1 + + + def test_keep_housenumbers_from_placex_table_hnr_list(self, add_housenumber, word_table, placex_table): + add_housenumber(9991, '9 b') + add_housenumber(9990, '34z') + placex_table.add(housenumber='9 a;9 b;9 c') + + assert word_table.count_housenumbers() == 2 + self.tok.update_word_tokens() + assert word_table.count_housenumbers() == 1