X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/80ea13437df4c6d57ea503adbdfc9928de8d859c..fa1c402b47bdd1f76b8528885c0e3a5e90731cdc:/test/python/tokenizer/test_icu.py diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py index 6138a03a..357b7d4a 100644 --- a/test/python/tokenizer/test_icu.py +++ b/test/python/tokenizer/test_icu.py @@ -1,23 +1,22 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: GPL-3.0-or-later # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2024 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for ICU tokenizer. """ -import shutil import yaml import itertools import pytest -from nominatim.tokenizer import icu_tokenizer -import nominatim.tokenizer.icu_rule_loader -from nominatim.db import properties -from nominatim.db.sql_preprocessor import SQLPreprocessor -from nominatim.indexer.place_info import PlaceInfo +from nominatim_db.tokenizer import icu_tokenizer +import nominatim_db.tokenizer.icu_rule_loader +from nominatim_db.db import properties +from nominatim_db.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.data.place_info import PlaceInfo from mock_icu_word_table import MockIcuWordTable @@ -32,8 +31,6 @@ def test_config(project_env, tmp_path): sqldir.mkdir() (sqldir / 'tokenizer').mkdir() (sqldir / 'tokenizer' / 'icu_tokenizer.sql').write_text("SELECT 'a'") - shutil.copy(str(project_env.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'), - str(sqldir / 'tokenizer' / 'icu_tokenizer_tables.sql')) project_env.lib_dir.sql = sqldir @@ -72,7 +69,8 @@ def analyzer(tokenizer_factory, test_config, monkeypatch, def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',), variants=('~gasse -> gasse', 'street => st', ), - sanitizers=[], with_housenumber=False): + sanitizers=[], with_housenumber=False, + with_postcode=False): cfgstr = {'normalization': list(norm), 'sanitizers': sanitizers, 'transliteration': list(trans), @@ -81,8 +79,11 @@ def analyzer(tokenizer_factory, test_config, monkeypatch, if with_housenumber: cfgstr['token-analysis'].append({'id': '@housenumber', 'analyzer': 'housenumbers'}) + if with_postcode: + cfgstr['token-analysis'].append({'id': '@postcode', + 'analyzer': 'postcodes'}) (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr)) - tok.loader = nominatim.tokenizer.icu_rule_loader.ICURuleLoader(test_config) + tok.loader = nominatim_db.tokenizer.icu_rule_loader.ICURuleLoader(test_config) return tok.name_analyzer() @@ -156,7 +157,7 @@ def test_init_new(tokenizer_factory, test_config, db_prop): tok = tokenizer_factory() tok.init_new_db(test_config) - assert db_prop(nominatim.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES) \ + assert db_prop(nominatim_db.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES) \ .startswith(':: lower ();') @@ -200,16 +201,14 @@ def test_update_sql_functions(db_prop, temp_db_cursor, def test_finalize_import(tokenizer_factory, temp_db_conn, temp_db_cursor, test_config, sql_preprocessor_cfg): - func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql' - func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT - AS $$ SELECT 'b'::text $$ LANGUAGE SQL""") - tok = tokenizer_factory() tok.init_new_db(test_config) + assert not temp_db_conn.index_exists('idx_word_word_id') + tok.finalize_import(test_config) - temp_db_cursor.scalar('SELECT test()') == 'b' + assert temp_db_conn.index_exists('idx_word_word_id') def test_check_database(test_config, tokenizer_factory, @@ -220,23 +219,28 @@ def test_check_database(test_config, tokenizer_factory, assert tok.check_database(test_config) is None -def test_update_statistics_reverse_only(word_table, tokenizer_factory): +def test_update_statistics_reverse_only(word_table, tokenizer_factory, test_config): tok = tokenizer_factory() - tok.update_statistics() + tok.update_statistics(test_config) -def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory): +def test_update_statistics(word_table, table_factory, temp_db_cursor, + tokenizer_factory, test_config): word_table.add_full_word(1000, 'hello') + word_table.add_full_word(1001, 'bye') table_factory('search_name', - 'place_id BIGINT, name_vector INT[]', - [(12, [1000])]) + 'place_id BIGINT, name_vector INT[], nameaddress_vector INT[]', + [(12, [1000], [1001])]) tok = tokenizer_factory() - tok.update_statistics() + tok.update_statistics(test_config) assert temp_db_cursor.scalar("""SELECT count(*) FROM word - WHERE type = 'W' and - (info->>'count')::int > 0""") > 0 + WHERE type = 'W' and word_id = 1000 and + (info->>'count')::int > 0""") == 1 + assert temp_db_cursor.scalar("""SELECT count(*) FROM word + WHERE type = 'W' and word_id = 1001 and + (info->>'addr_count')::int > 0""") == 1 def test_normalize_postcode(analyzer): @@ -246,28 +250,69 @@ def test_normalize_postcode(analyzer): anl.normalize_postcode('38 Б') == '38 Б' -def test_update_postcodes_from_db_empty(analyzer, table_factory, word_table): - table_factory('location_postcode', 'postcode TEXT', - content=(('1234',), ('12 34',), ('AB23',), ('1234',))) +class TestPostcodes: - with analyzer() as anl: - anl.update_postcodes_from_db() + @pytest.fixture(autouse=True) + def setup(self, analyzer, sql_functions): + sanitizers = [{'step': 'clean-postcodes'}] + with analyzer(sanitizers=sanitizers, with_postcode=True) as anl: + self.analyzer = anl + yield anl - assert word_table.count() == 3 - assert word_table.get_postcodes() == {'1234', '12 34', 'AB23'} + def process_postcode(self, cc, postcode): + return self.analyzer.process_place(PlaceInfo({'country_code': cc, + 'address': {'postcode': postcode}})) -def test_update_postcodes_from_db_add_and_remove(analyzer, table_factory, word_table): - table_factory('location_postcode', 'postcode TEXT', - content=(('1234',), ('45BC', ), ('XX45', ))) - word_table.add_postcode(' 1234', '1234') - word_table.add_postcode(' 5678', '5678') - with analyzer() as anl: - anl.update_postcodes_from_db() + def test_update_postcodes_from_db_empty(self, table_factory, word_table): + table_factory('location_postcode', 'country_code TEXT, postcode TEXT', + content=(('de', '12345'), ('se', '132 34'), + ('bm', 'AB23'), ('fr', '12345'))) + + self.analyzer.update_postcodes_from_db() + + assert word_table.count() == 5 + assert word_table.get_postcodes() == {'12345', '132 34@132 34', 'AB 23@AB 23'} + + + def test_update_postcodes_from_db_ambigious(self, table_factory, word_table): + table_factory('location_postcode', 'country_code TEXT, postcode TEXT', + content=(('in', '123456'), ('sg', '123456'))) + + self.analyzer.update_postcodes_from_db() + + assert word_table.count() == 3 + assert word_table.get_postcodes() == {'123456', '123456@123 456'} + + + def test_update_postcodes_from_db_add_and_remove(self, table_factory, word_table): + table_factory('location_postcode', 'country_code TEXT, postcode TEXT', + content=(('ch', '1234'), ('bm', 'BC 45'), ('bm', 'XX45'))) + word_table.add_postcode(' 1234', '1234') + word_table.add_postcode(' 5678', '5678') + + self.analyzer.update_postcodes_from_db() + + assert word_table.count() == 5 + assert word_table.get_postcodes() == {'1234', 'BC 45@BC 45', 'XX 45@XX 45'} + + + def test_process_place_postcode_simple(self, word_table): + info = self.process_postcode('de', '12345') + + assert info['postcode'] == '12345' + + assert word_table.get_postcodes() == {'12345', } + + + def test_process_place_postcode_with_space(self, word_table): + info = self.process_postcode('in', '123 567') + + assert info['postcode'] == '123567' + + assert word_table.get_postcodes() == {'123567@123 567', } - assert word_table.count() == 3 - assert word_table.get_postcodes() == {'1234', '45BC', 'XX45'} def test_update_special_phrase_empty_table(analyzer, word_table): @@ -478,7 +523,7 @@ class TestPlaceAddress: def test_process_place_nonexisting_street(self): info = self.process_address(street='Grand Road') - assert 'street' not in info + assert info['street'] == '{}' def test_process_place_multiple_street_tags(self): @@ -493,7 +538,7 @@ class TestPlaceAddress: def test_process_place_street_empty(self): info = self.process_address(street='🜵') - assert 'street' not in info + assert info['street'] == '{}' def test_process_place_street_from_cache(self): @@ -509,7 +554,7 @@ class TestPlaceAddress: def test_process_place_place(self): info = self.process_address(place='Honu Lulu') - assert eval(info['place']) == self.name_token_set('HONU', 'LULU') + assert eval(info['place']) == self.name_token_set('HONU', 'LULU', '#HONU LULU') def test_process_place_place_extra(self): @@ -529,8 +574,8 @@ class TestPlaceAddress: suburb='Zwickau', street='Hauptstr', full='right behind the church') - city = self.name_token_set('ZWICKAU') - state = self.name_token_set('SACHSEN') + city = self.name_token_set('ZWICKAU', '#ZWICKAU') + state = self.name_token_set('SACHSEN', '#SACHSEN') result = {k: eval(v) for k,v in info['addr'].items()} @@ -542,7 +587,7 @@ class TestPlaceAddress: result = {k: eval(v) for k,v in info['addr'].items()} - assert result == {'city': self.name_token_set('Bruxelles')} + assert result == {'city': self.name_token_set('Bruxelles', '#Bruxelles')} def test_process_place_address_terms_empty(self):