+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
"""
-Tests for Legacy ICU tokenizer.
+Tests for ICU tokenizer.
"""
import shutil
import yaml
+import itertools
import pytest
@pytest.fixture
-def test_config(def_config, tmp_path):
- def_config.project_dir = tmp_path / 'project'
- def_config.project_dir.mkdir()
-
+def test_config(project_env, tmp_path):
sqldir = tmp_path / 'sql'
sqldir.mkdir()
(sqldir / 'tokenizer').mkdir()
(sqldir / 'tokenizer' / 'icu_tokenizer.sql').write_text("SELECT 'a'")
- shutil.copy(str(def_config.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'),
+ shutil.copy(str(project_env.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'),
str(sqldir / 'tokenizer' / 'icu_tokenizer_tables.sql'))
- def_config.lib_dir.sql = sqldir
+ project_env.lib_dir.sql = sqldir
- return def_config
+ return project_env
@pytest.fixture
""")
-@pytest.fixture
-def getorcreate_hnr_id(temp_db_cursor):
- temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT)
- RETURNS INTEGER AS $$
- SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
-
def test_init_new(tokenizer_factory, test_config, db_prop):
tok = tokenizer_factory()
assert test_content == set((('1133', ), ))
+def test_finalize_import(tokenizer_factory, temp_db_conn,
+ temp_db_cursor, test_config, sql_preprocessor_cfg):
+ func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql'
+ func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT
+ AS $$ SELECT 'b'::text $$ LANGUAGE SQL""")
+
+ tok = tokenizer_factory()
+ tok.init_new_db(test_config)
+
+ tok.finalize_import(test_config)
+
+ temp_db_cursor.scalar('SELECT test()') == 'b'
+
+
+def test_check_database(test_config, tokenizer_factory,
+ temp_db_cursor, sql_preprocessor_cfg):
+ tok = tokenizer_factory()
+ tok.init_new_db(test_config)
+
+ assert tok.check_database(test_config) is None
+
+
+def test_update_statistics_reverse_only(word_table, tokenizer_factory):
+ tok = tokenizer_factory()
+ tok.update_statistics()
+
+
+def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory):
+ word_table.add_full_word(1000, 'hello')
+ table_factory('search_name',
+ 'place_id BIGINT, name_vector INT[]',
+ [(12, [1000])])
+ tok = tokenizer_factory()
+
+ tok.update_statistics()
+
+ assert temp_db_cursor.scalar("""SELECT count(*) FROM word
+ WHERE type = 'W' and
+ (info->>'count')::int > 0""") > 0
+
+
def test_normalize_postcode(analyzer):
with analyzer() as anl:
anl.normalize_postcode('123') == '123'
@pytest.fixture(autouse=True)
def setup(self, analyzer, sql_functions):
- with analyzer(trans=(":: upper()", "'🜵' > ' '")) as anl:
+ hnr = {'step': 'clean-housenumbers',
+ 'filter-kind': ['housenumber', 'conscriptionnumber', 'streetnumber']}
+ with analyzer(trans=(":: upper()", "'🜵' > ' '"), sanitizers=[hnr]) as anl:
self.analyzer = anl
yield anl
+ @pytest.fixture
+ def getorcreate_hnr_id(self, temp_db_cursor):
+ temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT)
+ RETURNS INTEGER AS $$
+ SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
+
+
def process_address(self, **kwargs):
return self.analyzer.process_place(PlaceInfo({'address': kwargs}))
assert info['hnr_tokens'] == "{-1}"
- def test_process_place_housenumbers_lists(self, getorcreate_hnr_id):
- info = self.process_address(conscriptionnumber='1; 2;3')
-
- assert set(info['hnr'].split(';')) == set(('1', '2', '3'))
- assert info['hnr_tokens'] == "{-1,-2,-3}"
-
-
def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id):
info = self.process_address(housenumber='134',
conscriptionnumber='134',
def test_process_place_street(self):
+ self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road'}}))
+ info = self.process_address(street='Grand Road')
+
+ assert eval(info['street']) == self.name_token_set('#Grand Road')
+
+
+ def test_process_place_nonexisting_street(self):
info = self.process_address(street='Grand Road')
- assert eval(info['street']) == self.name_token_set('GRAND', 'ROAD')
+ assert 'street' not in info
+
+
+ def test_process_place_multiple_street_tags(self):
+ self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road',
+ 'ref': '05989'}}))
+ info = self.process_address(**{'street': 'Grand Road',
+ 'street:sym_ul': '05989'})
+
+ assert eval(info['street']) == self.name_token_set('#Grand Road', '#05989')
def test_process_place_street_empty(self):
assert 'street' not in info
+ def test_process_place_street_from_cache(self):
+ self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road'}}))
+ self.process_address(street='Grand Road')
+
+ # request address again
+ info = self.process_address(street='Grand Road')
+
+ assert eval(info['street']) == self.name_token_set('#Grand Road')
+
+
def test_process_place_place(self):
info = self.process_address(place='Honu Lulu')
assert eval(info['place']) == self.name_token_set('HONU', 'LULU')
+ def test_process_place_place_extra(self):
+ info = self.process_address(**{'place:en': 'Honu Lulu'})
+
+ assert 'place' not in info
+
+
def test_process_place_place_empty(self):
info = self.process_address(place='🜵')
assert result == {'city': city, 'suburb': city, 'state': state}
+ def test_process_place_multiple_address_terms(self):
+ info = self.process_address(**{'city': 'Bruxelles', 'city:de': 'Brüssel'})
+
+ result = {k: eval(v) for k,v in info['addr'].items()}
+
+ assert result == {'city': self.name_token_set('Bruxelles')}
+
+
def test_process_place_address_terms_empty(self):
info = self.process_address(country='de', city=' ', street='Hauptstr',
full='right behind the church')
assert 'addr' not in info
+
+class TestUpdateWordTokens:
+
+ @pytest.fixture(autouse=True)
+ def setup(self, tokenizer_factory, table_factory, placex_table, word_table):
+ table_factory('search_name', 'place_id BIGINT, name_vector INT[]')
+ self.tok = tokenizer_factory()
+
+
+ @pytest.fixture
+ def search_entry(self, temp_db_cursor):
+ place_id = itertools.count(1000)
+
+ def _insert(*args):
+ temp_db_cursor.execute("INSERT INTO search_name VALUES (%s, %s)",
+ (next(place_id), list(args)))
+
+ return _insert
+
+
+ @pytest.mark.parametrize('hnr', ('1a', '1234567', '34 5'))
+ def test_remove_unused_housenumbers(self, word_table, hnr):
+ word_table.add_housenumber(1000, hnr)
+
+ assert word_table.count_housenumbers() == 1
+ self.tok.update_word_tokens()
+ assert word_table.count_housenumbers() == 0
+
+
+ def test_keep_unused_numeral_housenumbers(self, word_table):
+ word_table.add_housenumber(1000, '5432')
+
+ assert word_table.count_housenumbers() == 1
+ self.tok.update_word_tokens()
+ assert word_table.count_housenumbers() == 1
+
+
+ def test_keep_housenumbers_from_search_name_table(self, word_table, search_entry):
+ word_table.add_housenumber(9999, '5432a')
+ word_table.add_housenumber(9991, '9 a')
+ search_entry(123, 9999, 34)
+
+ assert word_table.count_housenumbers() == 2
+ self.tok.update_word_tokens()
+ assert word_table.count_housenumbers() == 1
+
+
+ def test_keep_housenumbers_from_placex_table(self, word_table, placex_table):
+ word_table.add_housenumber(9999, '5432a')
+ word_table.add_housenumber(9990, '34z')
+ placex_table.add(housenumber='34z')
+ placex_table.add(housenumber='25432a')
+
+ assert word_table.count_housenumbers() == 2
+ self.tok.update_word_tokens()
+ assert word_table.count_housenumbers() == 1
+
+
+ def test_keep_housenumbers_from_placex_table_hnr_list(self, word_table, placex_table):
+ word_table.add_housenumber(9991, '9 b')
+ word_table.add_housenumber(9990, '34z')
+ placex_table.add(housenumber='9 a;9 b;9 c')
+
+ assert word_table.count_housenumbers() == 2
+ self.tok.update_word_tokens()
+ assert word_table.count_housenumbers() == 1
projects / nominatim.git / blobdiff