X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/336258ecf82548a46715b7165b0547dacc161e07..824562357b33b009b93286aaf109320fa010520a:/test/python/test_tokenizer_icu.py diff --git a/test/python/test_tokenizer_icu.py b/test/python/test_tokenizer_icu.py index b7101c3f..4d029dec 100644 --- a/test/python/test_tokenizer_icu.py +++ b/test/python/test_tokenizer_icu.py @@ -7,9 +7,10 @@ import yaml import pytest from nominatim.tokenizer import icu_tokenizer -from nominatim.tokenizer.icu_name_processor import ICUNameProcessorRules from nominatim.tokenizer.icu_rule_loader import ICURuleLoader from nominatim.db import properties +from nominatim.db.sql_preprocessor import SQLPreprocessor +from nominatim.indexer.place_info import PlaceInfo from mock_icu_word_table import MockIcuWordTable @@ -66,16 +67,29 @@ def analyzer(tokenizer_factory, test_config, monkeypatch, monkeypatch.undo() def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',), - variants=('~gasse -> gasse', 'street => st', )): - cfgstr = {'normalization' : list(norm), - 'transliteration' : list(trans), - 'variants' : [ {'words': list(variants)}]} - tok.naming_rules = ICUNameProcessorRules(loader=ICURuleLoader(cfgstr)) + variants=('~gasse -> gasse', 'street => st', ), + sanitizers=[]): + cfgstr = {'normalization': list(norm), + 'sanitizers': sanitizers, + 'transliteration': list(trans), + 'token-analysis': [{'analyzer': 'generic', + 'variants': [{'words': list(variants)}]}]} + (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr)) + tok.loader = ICURuleLoader(test_config) return tok.name_analyzer() return _mk_analyser +@pytest.fixture +def sql_functions(temp_db_conn, def_config, src_dir): + orig_sql = def_config.lib_dir.sql + def_config.lib_dir.sql = src_dir / 'lib-sql' + sqlproc = SQLPreprocessor(temp_db_conn, def_config) + sqlproc.run_sql_file(temp_db_conn, 'functions/utils.sql') + sqlproc.run_sql_file(temp_db_conn, 'tokenizer/icu_tokenizer.sql') + def_config.lib_dir.sql = orig_sql + @pytest.fixture def getorcreate_full_word(temp_db_cursor): @@ -144,10 +158,9 @@ def test_init_new(tokenizer_factory, test_config, monkeypatch, db_prop): tok.init_new_db(test_config) assert db_prop(icu_tokenizer.DBCFG_TERM_NORMALIZATION) == ':: lower();' - assert db_prop(icu_tokenizer.DBCFG_MAXWORDFREQ) is not None -def test_init_word_table(tokenizer_factory, test_config, place_row, word_table): +def test_init_word_table(tokenizer_factory, test_config, place_row, temp_db_cursor): place_row(names={'name' : 'Test Area', 'ref' : '52'}) place_row(names={'name' : 'No Area'}) place_row(names={'name' : 'Holzstrasse'}) @@ -155,41 +168,32 @@ def test_init_word_table(tokenizer_factory, test_config, place_row, word_table): tok = tokenizer_factory() tok.init_new_db(test_config) - assert word_table.get_partial_words() == {('test', 1), - ('no', 1), ('area', 2), - ('holz', 1), ('strasse', 1), - ('str', 1)} + assert temp_db_cursor.table_exists('word') def test_init_from_project(monkeypatch, test_config, tokenizer_factory): monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();') - monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '90300') tok = tokenizer_factory() tok.init_new_db(test_config) monkeypatch.undo() tok = tokenizer_factory() - tok.init_from_project() + tok.init_from_project(test_config) - assert tok.naming_rules is not None + assert tok.loader is not None assert tok.term_normalization == ':: lower();' - assert tok.max_word_frequency == '90300' def test_update_sql_functions(db_prop, temp_db_cursor, tokenizer_factory, test_config, table_factory, monkeypatch): - monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '1133') tok = tokenizer_factory() tok.init_new_db(test_config) - monkeypatch.undo() - - assert db_prop(icu_tokenizer.DBCFG_MAXWORDFREQ) == '1133' table_factory('test', 'txt TEXT') func_file = test_config.lib_dir.sql / 'tokenizer' / 'icu_tokenizer.sql' - func_file.write_text("""INSERT INTO test VALUES ('{{max_word_freq}}')""") + func_file.write_text("""INSERT INTO test VALUES (1133)""") tok.update_sql_functions(test_config) @@ -304,45 +308,55 @@ def test_add_country_names_extend(analyzer, word_table): class TestPlaceNames: @pytest.fixture(autouse=True) - def setup(self, analyzer, getorcreate_full_word): - with analyzer() as anl: + def setup(self, analyzer, sql_functions): + sanitizers = [{'step': 'split-name-list'}, + {'step': 'strip-brace-terms'}] + with analyzer(sanitizers=sanitizers) as anl: self.analyzer = anl yield anl def expect_name_terms(self, info, *expected_terms): tokens = self.analyzer.get_word_token_info(expected_terms) - print (tokens) for token in tokens: assert token[2] is not None, "No token for {0}".format(token) assert eval(info['names']) == set((t[2] for t in tokens)) + def process_named_place(self, names): + return self.analyzer.process_place(PlaceInfo({'name': names})) + + def test_simple_names(self): - info = self.analyzer.process_place({'name': {'name': 'Soft bAr', 'ref': '34'}}) + info = self.process_named_place({'name': 'Soft bAr', 'ref': '34'}) self.expect_name_terms(info, '#Soft bAr', '#34', 'Soft', 'bAr', '34') @pytest.mark.parametrize('sep', [',' , ';']) def test_names_with_separator(self, sep): - info = self.analyzer.process_place({'name': {'name': sep.join(('New York', 'Big Apple'))}}) + info = self.process_named_place({'name': sep.join(('New York', 'Big Apple'))}) self.expect_name_terms(info, '#New York', '#Big Apple', 'new', 'york', 'big', 'apple') def test_full_names_with_bracket(self): - info = self.analyzer.process_place({'name': {'name': 'Houseboat (left)'}}) + info = self.process_named_place({'name': 'Houseboat (left)'}) self.expect_name_terms(info, '#Houseboat (left)', '#Houseboat', 'houseboat', 'left') def test_country_name(self, word_table): - info = self.analyzer.process_place({'name': {'name': 'Norge'}, - 'country_feature': 'no'}) + place = PlaceInfo({'name' : {'name': 'Norge'}, + 'country_code': 'no', + 'rank_address': 4, + 'class': 'boundary', + 'type': 'administrative'}) + + info = self.analyzer.process_place(place) self.expect_name_terms(info, '#norge', 'norge') assert word_table.get_country() == {('no', 'NORGE')} @@ -351,14 +365,14 @@ class TestPlaceNames: class TestPlaceAddress: @pytest.fixture(autouse=True) - def setup(self, analyzer, getorcreate_full_word): + def setup(self, analyzer, sql_functions): with analyzer(trans=(":: upper()", "'🜵' > ' '")) as anl: self.analyzer = anl yield anl def process_address(self, **kwargs): - return self.analyzer.process_place({'address': kwargs}) + return self.analyzer.process_place(PlaceInfo({'address': kwargs})) def name_token_set(self, *expected_terms): @@ -424,7 +438,7 @@ class TestPlaceAddress: def test_process_place_street(self): info = self.process_address(street='Grand Road') - assert eval(info['street']) == self.name_token_set('#GRAND ROAD') + assert eval(info['street']) == self.name_token_set('GRAND', 'ROAD') def test_process_place_street_empty(self): @@ -436,16 +450,13 @@ class TestPlaceAddress: def test_process_place_place(self): info = self.process_address(place='Honu Lulu') - assert eval(info['place_search']) == self.name_token_set('#HONU LULU', - 'HONU', 'LULU') - assert eval(info['place_match']) == self.name_token_set('#HONU LULU') + assert eval(info['place']) == self.name_token_set('HONU', 'LULU') def test_process_place_place_empty(self): info = self.process_address(place='🜵') - assert 'place_search' not in info - assert 'place_match' not in info + assert 'place' not in info def test_process_place_address_terms(self): @@ -453,16 +464,12 @@ class TestPlaceAddress: suburb='Zwickau', street='Hauptstr', full='right behind the church') - city_full = self.name_token_set('#ZWICKAU') - city_all = self.name_token_set('#ZWICKAU', 'ZWICKAU') - state_full = self.name_token_set('#SACHSEN') - state_all = self.name_token_set('#SACHSEN', 'SACHSEN') + city = self.name_token_set('ZWICKAU') + state = self.name_token_set('SACHSEN') - result = {k: [eval(v[0]), eval(v[1])] for k,v in info['addr'].items()} + result = {k: eval(v) for k,v in info['addr'].items()} - assert result == {'city': [city_all, city_full], - 'suburb': [city_all, city_full], - 'state': [state_all, state_full]} + assert result == {'city': city, 'suburb': city, 'state': state} def test_process_place_address_terms_empty(self):