X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/b45f761227ffed8eb328df7a5d9e979197b8c690..8f3845660f18bdbf2dd42dd2c6db6c7fa5160f3e:/test/python/tokenizer/test_icu.py?ds=inline diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py index 2d9da69a..2a4865db 100644 --- a/test/python/tokenizer/test_icu.py +++ b/test/python/tokenizer/test_icu.py @@ -7,7 +7,6 @@ """ Tests for ICU tokenizer. """ -import shutil import yaml import itertools @@ -32,8 +31,6 @@ def test_config(project_env, tmp_path): sqldir.mkdir() (sqldir / 'tokenizer').mkdir() (sqldir / 'tokenizer' / 'icu_tokenizer.sql').write_text("SELECT 'a'") - shutil.copy(str(project_env.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'), - str(sqldir / 'tokenizer' / 'icu_tokenizer_tables.sql')) project_env.lib_dir.sql = sqldir @@ -204,16 +201,14 @@ def test_update_sql_functions(db_prop, temp_db_cursor, def test_finalize_import(tokenizer_factory, temp_db_conn, temp_db_cursor, test_config, sql_preprocessor_cfg): - func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql' - func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT - AS $$ SELECT 'b'::text $$ LANGUAGE SQL""") - tok = tokenizer_factory() tok.init_new_db(test_config) + assert not temp_db_conn.index_exists('idx_word_word_id') + tok.finalize_import(test_config) - temp_db_cursor.scalar('SELECT test()') == 'b' + assert temp_db_conn.index_exists('idx_word_word_id') def test_check_database(test_config, tokenizer_factory, @@ -224,23 +219,28 @@ def test_check_database(test_config, tokenizer_factory, assert tok.check_database(test_config) is None -def test_update_statistics_reverse_only(word_table, tokenizer_factory): +def test_update_statistics_reverse_only(word_table, tokenizer_factory, test_config): tok = tokenizer_factory() - tok.update_statistics() + tok.update_statistics(test_config) -def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory): +def test_update_statistics(word_table, table_factory, temp_db_cursor, + tokenizer_factory, test_config): word_table.add_full_word(1000, 'hello') + word_table.add_full_word(1001, 'bye') table_factory('search_name', - 'place_id BIGINT, name_vector INT[]', - [(12, [1000])]) + 'place_id BIGINT, name_vector INT[], nameaddress_vector INT[]', + [(12, [1000], [1001])]) tok = tokenizer_factory() - tok.update_statistics() + tok.update_statistics(test_config) assert temp_db_cursor.scalar("""SELECT count(*) FROM word - WHERE type = 'W' and - (info->>'count')::int > 0""") > 0 + WHERE type = 'W' and word_id = 1000 and + (info->>'count')::int > 0""") == 1 + assert temp_db_cursor.scalar("""SELECT count(*) FROM word + WHERE type = 'W' and word_id = 1001 and + (info->>'addr_count')::int > 0""") == 1 def test_normalize_postcode(analyzer): @@ -554,7 +554,7 @@ class TestPlaceAddress: def test_process_place_place(self): info = self.process_address(place='Honu Lulu') - assert eval(info['place']) == self.name_token_set('HONU', 'LULU') + assert eval(info['place']) == self.name_token_set('HONU', 'LULU', '#HONU LULU') def test_process_place_place_extra(self): @@ -574,8 +574,8 @@ class TestPlaceAddress: suburb='Zwickau', street='Hauptstr', full='right behind the church') - city = self.name_token_set('ZWICKAU') - state = self.name_token_set('SACHSEN') + city = self.name_token_set('ZWICKAU', '#ZWICKAU') + state = self.name_token_set('SACHSEN', '#SACHSEN') result = {k: eval(v) for k,v in info['addr'].items()} @@ -587,7 +587,7 @@ class TestPlaceAddress: result = {k: eval(v) for k,v in info['addr'].items()} - assert result == {'city': self.name_token_set('Bruxelles')} + assert result == {'city': self.name_token_set('Bruxelles', '#Bruxelles')} def test_process_place_address_terms_empty(self):