X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/09c9fad6c31733065492306c9ee3c052e50ac115..109af0ef1c55421a166fd2d45586d04fe7f6f969:/test/python/test_tokenizer_icu.py diff --git a/test/python/test_tokenizer_icu.py b/test/python/test_tokenizer_icu.py index ed079269..5dbe292e 100644 --- a/test/python/test_tokenizer_icu.py +++ b/test/python/test_tokenizer_icu.py @@ -7,10 +7,10 @@ import yaml import pytest from nominatim.tokenizer import icu_tokenizer -from nominatim.tokenizer.icu_name_processor import ICUNameProcessorRules -from nominatim.tokenizer.icu_rule_loader import ICURuleLoader +import nominatim.tokenizer.icu_rule_loader from nominatim.db import properties from nominatim.db.sql_preprocessor import SQLPreprocessor +from nominatim.indexer.place_info import PlaceInfo from mock_icu_word_table import MockIcuWordTable @@ -67,11 +67,15 @@ def analyzer(tokenizer_factory, test_config, monkeypatch, monkeypatch.undo() def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',), - variants=('~gasse -> gasse', 'street => st', )): - cfgstr = {'normalization' : list(norm), - 'transliteration' : list(trans), - 'variants' : [ {'words': list(variants)}]} - tok.naming_rules = ICUNameProcessorRules(loader=ICURuleLoader(cfgstr)) + variants=('~gasse -> gasse', 'street => st', ), + sanitizers=[]): + cfgstr = {'normalization': list(norm), + 'sanitizers': sanitizers, + 'transliteration': list(trans), + 'token-analysis': [{'analyzer': 'generic', + 'variants': [{'words': list(variants)}]}]} + (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr)) + tok.loader = nominatim.tokenizer.icu_rule_loader.ICURuleLoader(test_config) return tok.name_analyzer() @@ -147,16 +151,15 @@ def getorcreate_hnr_id(temp_db_cursor): SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""") -def test_init_new(tokenizer_factory, test_config, monkeypatch, db_prop): - monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();') - +def test_init_new(tokenizer_factory, test_config, db_prop): tok = tokenizer_factory() tok.init_new_db(test_config) - assert db_prop(icu_tokenizer.DBCFG_TERM_NORMALIZATION) == ':: lower();' + assert db_prop(nominatim.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES) \ + .startswith(':: lower ();') -def test_init_word_table(tokenizer_factory, test_config, place_row, word_table): +def test_init_word_table(tokenizer_factory, test_config, place_row, temp_db_cursor): place_row(names={'name' : 'Test Area', 'ref' : '52'}) place_row(names={'name' : 'No Area'}) place_row(names={'name' : 'Holzstrasse'}) @@ -164,23 +167,17 @@ def test_init_word_table(tokenizer_factory, test_config, place_row, word_table): tok = tokenizer_factory() tok.init_new_db(test_config) - assert word_table.get_partial_words() == {('test', 1), - ('no', 1), ('area', 2), - ('holz', 1), ('strasse', 1), - ('str', 1)} + assert temp_db_cursor.table_exists('word') -def test_init_from_project(monkeypatch, test_config, tokenizer_factory): - monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();') +def test_init_from_project(test_config, tokenizer_factory): tok = tokenizer_factory() tok.init_new_db(test_config) - monkeypatch.undo() tok = tokenizer_factory() - tok.init_from_project() + tok.init_from_project(test_config) - assert tok.naming_rules is not None - assert tok.term_normalization == ':: lower();' + assert tok.loader is not None def test_update_sql_functions(db_prop, temp_db_cursor, @@ -308,44 +305,54 @@ class TestPlaceNames: @pytest.fixture(autouse=True) def setup(self, analyzer, sql_functions): - with analyzer() as anl: + sanitizers = [{'step': 'split-name-list'}, + {'step': 'strip-brace-terms'}] + with analyzer(sanitizers=sanitizers) as anl: self.analyzer = anl yield anl def expect_name_terms(self, info, *expected_terms): tokens = self.analyzer.get_word_token_info(expected_terms) - print (tokens) for token in tokens: assert token[2] is not None, "No token for {0}".format(token) assert eval(info['names']) == set((t[2] for t in tokens)) + def process_named_place(self, names): + return self.analyzer.process_place(PlaceInfo({'name': names})) + + def test_simple_names(self): - info = self.analyzer.process_place({'name': {'name': 'Soft bAr', 'ref': '34'}}) + info = self.process_named_place({'name': 'Soft bAr', 'ref': '34'}) self.expect_name_terms(info, '#Soft bAr', '#34', 'Soft', 'bAr', '34') @pytest.mark.parametrize('sep', [',' , ';']) def test_names_with_separator(self, sep): - info = self.analyzer.process_place({'name': {'name': sep.join(('New York', 'Big Apple'))}}) + info = self.process_named_place({'name': sep.join(('New York', 'Big Apple'))}) self.expect_name_terms(info, '#New York', '#Big Apple', 'new', 'york', 'big', 'apple') def test_full_names_with_bracket(self): - info = self.analyzer.process_place({'name': {'name': 'Houseboat (left)'}}) + info = self.process_named_place({'name': 'Houseboat (left)'}) self.expect_name_terms(info, '#Houseboat (left)', '#Houseboat', 'houseboat', 'left') def test_country_name(self, word_table): - info = self.analyzer.process_place({'name': {'name': 'Norge'}, - 'country_feature': 'no'}) + place = PlaceInfo({'name' : {'name': 'Norge'}, + 'country_code': 'no', + 'rank_address': 4, + 'class': 'boundary', + 'type': 'administrative'}) + + info = self.analyzer.process_place(place) self.expect_name_terms(info, '#norge', 'norge') assert word_table.get_country() == {('no', 'NORGE')} @@ -361,7 +368,7 @@ class TestPlaceAddress: def process_address(self, **kwargs): - return self.analyzer.process_place({'address': kwargs}) + return self.analyzer.process_place(PlaceInfo({'address': kwargs})) def name_token_set(self, *expected_terms):