import pytest
from nominatim.tokenizer import icu_tokenizer
-from nominatim.tokenizer.icu_name_processor import ICUNameProcessorRules
-from nominatim.tokenizer.icu_rule_loader import ICURuleLoader
+import nominatim.tokenizer.icu_rule_loader
from nominatim.db import properties
+from nominatim.db.sql_preprocessor import SQLPreprocessor
+from nominatim.indexer.place_info import PlaceInfo
from mock_icu_word_table import MockIcuWordTable
monkeypatch.undo()
def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',),
- variants=('~gasse -> gasse', 'street => st', )):
- cfgstr = {'normalization' : list(norm),
- 'transliteration' : list(trans),
- 'variants' : [ {'words': list(variants)}]}
- tok.naming_rules = ICUNameProcessorRules(loader=ICURuleLoader(cfgstr))
+ variants=('~gasse -> gasse', 'street => st', ),
+ sanitizers=[]):
+ cfgstr = {'normalization': list(norm),
+ 'sanitizers': sanitizers,
+ 'transliteration': list(trans),
+ 'token-analysis': [{'analyzer': 'generic',
+ 'variants': [{'words': list(variants)}]}]}
+ (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr))
+ tok.loader = nominatim.tokenizer.icu_rule_loader.ICURuleLoader(test_config)
return tok.name_analyzer()
return _mk_analyser
+@pytest.fixture
+def sql_functions(temp_db_conn, def_config, src_dir):
+ orig_sql = def_config.lib_dir.sql
+ def_config.lib_dir.sql = src_dir / 'lib-sql'
+ sqlproc = SQLPreprocessor(temp_db_conn, def_config)
+ sqlproc.run_sql_file(temp_db_conn, 'functions/utils.sql')
+ sqlproc.run_sql_file(temp_db_conn, 'tokenizer/icu_tokenizer.sql')
+ def_config.lib_dir.sql = orig_sql
+
@pytest.fixture
def getorcreate_full_word(temp_db_cursor):
SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
-def test_init_new(tokenizer_factory, test_config, monkeypatch, db_prop):
- monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();')
-
+def test_init_new(tokenizer_factory, test_config, db_prop):
tok = tokenizer_factory()
tok.init_new_db(test_config)
- assert db_prop(icu_tokenizer.DBCFG_TERM_NORMALIZATION) == ':: lower();'
- assert db_prop(icu_tokenizer.DBCFG_MAXWORDFREQ) is not None
+ assert db_prop(nominatim.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES) \
+ .startswith(':: lower ();')
-def test_init_word_table(tokenizer_factory, test_config, place_row, word_table):
+def test_init_word_table(tokenizer_factory, test_config, place_row, temp_db_cursor):
place_row(names={'name' : 'Test Area', 'ref' : '52'})
place_row(names={'name' : 'No Area'})
place_row(names={'name' : 'Holzstrasse'})
tok = tokenizer_factory()
tok.init_new_db(test_config)
- assert word_table.get_partial_words() == {('test', 1),
- ('no', 1), ('area', 2),
- ('holz', 1), ('strasse', 1),
- ('str', 1)}
+ assert temp_db_cursor.table_exists('word')
-def test_init_from_project(monkeypatch, test_config, tokenizer_factory):
- monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();')
- monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '90300')
+def test_init_from_project(test_config, tokenizer_factory):
tok = tokenizer_factory()
tok.init_new_db(test_config)
- monkeypatch.undo()
tok = tokenizer_factory()
- tok.init_from_project()
+ tok.init_from_project(test_config)
- assert tok.naming_rules is not None
- assert tok.term_normalization == ':: lower();'
- assert tok.max_word_frequency == '90300'
+ assert tok.loader is not None
def test_update_sql_functions(db_prop, temp_db_cursor,
tokenizer_factory, test_config, table_factory,
monkeypatch):
- monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '1133')
tok = tokenizer_factory()
tok.init_new_db(test_config)
- monkeypatch.undo()
-
- assert db_prop(icu_tokenizer.DBCFG_MAXWORDFREQ) == '1133'
table_factory('test', 'txt TEXT')
func_file = test_config.lib_dir.sql / 'tokenizer' / 'icu_tokenizer.sql'
- func_file.write_text("""INSERT INTO test VALUES ('{{max_word_freq}}')""")
+ func_file.write_text("""INSERT INTO test VALUES (1133)""")
tok.update_sql_functions(test_config)
class TestPlaceNames:
@pytest.fixture(autouse=True)
- def setup(self, analyzer, getorcreate_full_word):
- with analyzer() as anl:
+ def setup(self, analyzer, sql_functions):
+ sanitizers = [{'step': 'split-name-list'},
+ {'step': 'strip-brace-terms'}]
+ with analyzer(sanitizers=sanitizers) as anl:
self.analyzer = anl
yield anl
def expect_name_terms(self, info, *expected_terms):
tokens = self.analyzer.get_word_token_info(expected_terms)
- print (tokens)
for token in tokens:
assert token[2] is not None, "No token for {0}".format(token)
assert eval(info['names']) == set((t[2] for t in tokens))
+ def process_named_place(self, names):
+ return self.analyzer.process_place(PlaceInfo({'name': names}))
+
+
def test_simple_names(self):
- info = self.analyzer.process_place({'name': {'name': 'Soft bAr', 'ref': '34'}})
+ info = self.process_named_place({'name': 'Soft bAr', 'ref': '34'})
self.expect_name_terms(info, '#Soft bAr', '#34', 'Soft', 'bAr', '34')
@pytest.mark.parametrize('sep', [',' , ';'])
def test_names_with_separator(self, sep):
- info = self.analyzer.process_place({'name': {'name': sep.join(('New York', 'Big Apple'))}})
+ info = self.process_named_place({'name': sep.join(('New York', 'Big Apple'))})
self.expect_name_terms(info, '#New York', '#Big Apple',
'new', 'york', 'big', 'apple')
def test_full_names_with_bracket(self):
- info = self.analyzer.process_place({'name': {'name': 'Houseboat (left)'}})
+ info = self.process_named_place({'name': 'Houseboat (left)'})
self.expect_name_terms(info, '#Houseboat (left)', '#Houseboat',
'houseboat', 'left')
def test_country_name(self, word_table):
- info = self.analyzer.process_place({'name': {'name': 'Norge'},
- 'country_feature': 'no'})
+ place = PlaceInfo({'name' : {'name': 'Norge'},
+ 'country_code': 'no',
+ 'rank_address': 4,
+ 'class': 'boundary',
+ 'type': 'administrative'})
+
+ info = self.analyzer.process_place(place)
self.expect_name_terms(info, '#norge', 'norge')
assert word_table.get_country() == {('no', 'NORGE')}
class TestPlaceAddress:
@pytest.fixture(autouse=True)
- def setup(self, analyzer, getorcreate_full_word):
+ def setup(self, analyzer, sql_functions):
with analyzer(trans=(":: upper()", "'🜵' > ' '")) as anl:
self.analyzer = anl
yield anl
def process_address(self, **kwargs):
- return self.analyzer.process_place({'address': kwargs})
+ return self.analyzer.process_place(PlaceInfo({'address': kwargs}))
def name_token_set(self, *expected_terms):
def test_process_place_street(self):
info = self.process_address(street='Grand Road')
- assert eval(info['street']) == self.name_token_set('#GRAND ROAD')
+ assert eval(info['street']) == self.name_token_set('GRAND', 'ROAD')
def test_process_place_street_empty(self):
def test_process_place_place(self):
info = self.process_address(place='Honu Lulu')
- assert eval(info['place_search']) == self.name_token_set('#HONU LULU',
- 'HONU', 'LULU')
- assert eval(info['place_match']) == self.name_token_set('#HONU LULU')
+ assert eval(info['place']) == self.name_token_set('HONU', 'LULU')
def test_process_place_place_empty(self):
info = self.process_address(place='🜵')
- assert 'place_search' not in info
- assert 'place_match' not in info
+ assert 'place' not in info
def test_process_place_address_terms(self):
suburb='Zwickau', street='Hauptstr',
full='right behind the church')
- city_full = self.name_token_set('#ZWICKAU')
- city_all = self.name_token_set('#ZWICKAU', 'ZWICKAU')
- state_full = self.name_token_set('#SACHSEN')
- state_all = self.name_token_set('#SACHSEN', 'SACHSEN')
+ city = self.name_token_set('ZWICKAU')
+ state = self.name_token_set('SACHSEN')
- result = {k: [eval(v[0]), eval(v[1])] for k,v in info['addr'].items()}
+ result = {k: eval(v) for k,v in info['addr'].items()}
- assert result == {'city': [city_all, city_full],
- 'suburb': [city_all, city_full],
- 'state': [state_all, state_full]}
+ assert result == {'city': city, 'suburb': city, 'state': state}
def test_process_place_address_terms_empty(self):
projects / nominatim.git / blobdiff