from nominatim.tokenizer.icu_name_processor import ICUNameProcessorRules
from nominatim.tokenizer.icu_rule_loader import ICURuleLoader
from nominatim.db import properties
+from nominatim.db.sql_preprocessor import SQLPreprocessor
+from nominatim.indexer.place_info import PlaceInfo
from mock_icu_word_table import MockIcuWordTable
def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',),
variants=('~gasse -> gasse', 'street => st', )):
- cfgfile = tmp_path / 'analyser_test_config.yaml'
- with cfgfile.open('w') as stream:
- cfgstr = {'normalization' : list(norm),
- 'transliteration' : list(trans),
- 'variants' : [ {'words': list(variants)}]}
- yaml.dump(cfgstr, stream)
- tok.naming_rules = ICUNameProcessorRules(loader=ICURuleLoader(cfgfile))
+ cfgstr = {'normalization' : list(norm),
+ 'transliteration' : list(trans),
+ 'variants' : [ {'words': list(variants)}]}
+ tok.naming_rules = ICUNameProcessorRules(loader=ICURuleLoader(cfgstr))
return tok.name_analyzer()
return _mk_analyser
+@pytest.fixture
+def sql_functions(temp_db_conn, def_config, src_dir):
+ orig_sql = def_config.lib_dir.sql
+ def_config.lib_dir.sql = src_dir / 'lib-sql'
+ sqlproc = SQLPreprocessor(temp_db_conn, def_config)
+ sqlproc.run_sql_file(temp_db_conn, 'functions/utils.sql')
+ sqlproc.run_sql_file(temp_db_conn, 'tokenizer/icu_tokenizer.sql')
+ def_config.lib_dir.sql = orig_sql
+
@pytest.fixture
def getorcreate_full_word(temp_db_cursor):
tok.init_new_db(test_config)
assert db_prop(icu_tokenizer.DBCFG_TERM_NORMALIZATION) == ':: lower();'
- assert db_prop(icu_tokenizer.DBCFG_MAXWORDFREQ) is not None
def test_init_word_table(tokenizer_factory, test_config, place_row, word_table):
def test_init_from_project(monkeypatch, test_config, tokenizer_factory):
monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();')
- monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '90300')
tok = tokenizer_factory()
tok.init_new_db(test_config)
monkeypatch.undo()
assert tok.naming_rules is not None
assert tok.term_normalization == ':: lower();'
- assert tok.max_word_frequency == '90300'
def test_update_sql_functions(db_prop, temp_db_cursor,
tokenizer_factory, test_config, table_factory,
monkeypatch):
- monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '1133')
tok = tokenizer_factory()
tok.init_new_db(test_config)
- monkeypatch.undo()
-
- assert db_prop(icu_tokenizer.DBCFG_MAXWORDFREQ) == '1133'
table_factory('test', 'txt TEXT')
func_file = test_config.lib_dir.sql / 'tokenizer' / 'icu_tokenizer.sql'
- func_file.write_text("""INSERT INTO test VALUES ('{{max_word_freq}}')""")
+ func_file.write_text("""INSERT INTO test VALUES (1133)""")
tok.update_sql_functions(test_config)
class TestPlaceNames:
@pytest.fixture(autouse=True)
- def setup(self, analyzer, getorcreate_full_word):
+ def setup(self, analyzer, sql_functions):
with analyzer() as anl:
self.analyzer = anl
yield anl
assert eval(info['names']) == set((t[2] for t in tokens))
+ def process_named_place(self, names):
+ place = {'name': names}
+
+ return self.analyzer.process_place(PlaceInfo(place))
+
+
def test_simple_names(self):
- info = self.analyzer.process_place({'name': {'name': 'Soft bAr', 'ref': '34'}})
+ info = self.process_named_place({'name': 'Soft bAr', 'ref': '34'})
self.expect_name_terms(info, '#Soft bAr', '#34', 'Soft', 'bAr', '34')
@pytest.mark.parametrize('sep', [',' , ';'])
def test_names_with_separator(self, sep):
- info = self.analyzer.process_place({'name': {'name': sep.join(('New York', 'Big Apple'))}})
+ info = self.process_named_place({'name': sep.join(('New York', 'Big Apple'))})
self.expect_name_terms(info, '#New York', '#Big Apple',
'new', 'york', 'big', 'apple')
def test_full_names_with_bracket(self):
- info = self.analyzer.process_place({'name': {'name': 'Houseboat (left)'}})
+ info = self.process_named_place({'name': 'Houseboat (left)'})
self.expect_name_terms(info, '#Houseboat (left)', '#Houseboat',
'houseboat', 'left')
def test_country_name(self, word_table):
- info = self.analyzer.process_place({'name': {'name': 'Norge'},
- 'country_feature': 'no'})
+ place = PlaceInfo({'name' : {'name': 'Norge'},
+ 'country_code': 'no',
+ 'rank_address': 4,
+ 'class': 'boundary',
+ 'type': 'administrative'})
+
+ info = self.analyzer.process_place(place)
self.expect_name_terms(info, '#norge', 'norge')
assert word_table.get_country() == {('no', 'NORGE')}
class TestPlaceAddress:
@pytest.fixture(autouse=True)
- def setup(self, analyzer, getorcreate_full_word):
+ def setup(self, analyzer, sql_functions):
with analyzer(trans=(":: upper()", "'🜵' > ' '")) as anl:
self.analyzer = anl
yield anl
def process_address(self, **kwargs):
- return self.analyzer.process_place({'address': kwargs})
+ return self.analyzer.process_place(PlaceInfo({'address': kwargs}))
def name_token_set(self, *expected_terms):
def test_process_place_street(self):
info = self.process_address(street='Grand Road')
- assert eval(info['street']) == self.name_token_set('#GRAND ROAD')
+ assert eval(info['street']) == self.name_token_set('GRAND', 'ROAD')
def test_process_place_street_empty(self):
def test_process_place_place(self):
info = self.process_address(place='Honu Lulu')
- assert eval(info['place_search']) == self.name_token_set('#HONU LULU',
- 'HONU', 'LULU')
- assert eval(info['place_match']) == self.name_token_set('#HONU LULU')
+ assert eval(info['place']) == self.name_token_set('HONU', 'LULU')
def test_process_place_place_empty(self):
info = self.process_address(place='🜵')
- assert 'place_search' not in info
- assert 'place_match' not in info
+ assert 'place' not in info
def test_process_place_address_terms(self):
suburb='Zwickau', street='Hauptstr',
full='right behind the church')
- city_full = self.name_token_set('#ZWICKAU')
- city_all = self.name_token_set('#ZWICKAU', 'ZWICKAU')
- state_full = self.name_token_set('#SACHSEN')
- state_all = self.name_token_set('#SACHSEN', 'SACHSEN')
+ city = self.name_token_set('ZWICKAU')
+ state = self.name_token_set('SACHSEN')
- result = {k: [eval(v[0]), eval(v[1])] for k,v in info['addr'].items()}
+ result = {k: eval(v) for k,v in info['addr'].items()}
- assert result == {'city': [city_all, city_full],
- 'suburb': [city_all, city_full],
- 'state': [state_all, state_full]}
+ assert result == {'city': city, 'suburb': city, 'state': state}
def test_process_place_address_terms_empty(self):
projects / nominatim.git / blobdiff