lift restrictions on search with frequent terms slightly

[nominatim.git] / test / python / tokenizer / test_legacy.py

diff --git a/test/python/tokenizer/test_legacy.py b/test/python/tokenizer/test_legacy.py
index 45cd94aab27f1856d3d9ee0e238f9347a1e5e3c5..bf208c92dc8684eb346f8b1f129614978494f1a3 100644 (file)
--- a/test/python/tokenizer/test_legacy.py
+++ b/test/python/tokenizer/test_legacy.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2024 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Test for legacy tokenizer.
 """
 """
 Test for legacy tokenizer.
 """


@@ -6,10 +12,10 @@ import re
 
 import pytest
 
 
 import pytest
 

-from nominatim.indexer.place_info import PlaceInfo
-from nominatim.tokenizer import legacy_tokenizer
-from nominatim.db import properties
-from nominatim.errors import UsageError
+from nominatim_db.data.place_info import PlaceInfo
+from nominatim_db.tokenizer import legacy_tokenizer
+from nominatim_db.db import properties
+from nominatim_db.errors import UsageError

 
 from mock_legacy_word_table import MockLegacyWordTable
 
 
 from mock_legacy_word_table import MockLegacyWordTable
 


@@ -23,7 +29,7 @@ def word_table(temp_db_conn):
 def test_config(project_env, tmp_path):
     module_dir = tmp_path / 'module_src'
     module_dir.mkdir()
 def test_config(project_env, tmp_path):
     module_dir = tmp_path / 'module_src'
     module_dir.mkdir()

-    (module_dir / 'nominatim.so').write_text('TEST nomiantim.so')
+    (module_dir / 'nominatim.so').write_text('TEST nominatim.so')

 
     project_env.lib_dir.module = module_dir
 
 
     project_env.lib_dir.module = module_dir
 


@@ -115,7 +121,7 @@ def test_init_new(tokenizer_factory, test_config, monkeypatch,
     outfile = test_config.project_dir / 'module' / 'nominatim.so'
 
     assert outfile.exists()
     outfile = test_config.project_dir / 'module' / 'nominatim.so'
 
     assert outfile.exists()

-    assert outfile.read_text() == 'TEST nomiantim.so'
+    assert outfile.read_text() == 'TEST nominatim.so'

     assert outfile.stat().st_mode == 33261
 
 
     assert outfile.stat().st_mode == 33261
 
 


@@ -179,7 +185,7 @@ def test_finalize_import(tokenizer_factory, temp_db_conn,
 
     func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql'
     func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT
 
     func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql'
     func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT

-                            AS $$ SELECT 'b' $$ LANGUAGE SQL""")
+                            AS $$ SELECT 'b'::text $$ LANGUAGE SQL""")

 
     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 
     tok = tokenizer_factory()
     tok.init_new_db(test_config)


@@ -200,7 +206,7 @@ def test_migrate_database(tokenizer_factory, test_config, temp_db_conn, monkeypa
     outfile = test_config.project_dir / 'module' / 'nominatim.so'
 
     assert outfile.exists()
     outfile = test_config.project_dir / 'module' / 'nominatim.so'
 
     assert outfile.exists()

-    assert outfile.read_text() == 'TEST nomiantim.so'
+    assert outfile.read_text() == 'TEST nominatim.so'

     assert outfile.stat().st_mode == 33261
 
 
     assert outfile.stat().st_mode == 33261
 
 


@@ -227,30 +233,37 @@ def test_check_database_bad_setup(test_config, tokenizer_factory, monkeypatch,
 
     # Inject a bad transliteration.
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION make_standard_name(name TEXT)
 
     # Inject a bad transliteration.
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION make_standard_name(name TEXT)

-                              RETURNS TEXT AS $$ SELECT 'garbage'; $$ LANGUAGE SQL""")
+                              RETURNS TEXT AS $$ SELECT 'garbage'::text; $$ LANGUAGE SQL""")

 
     assert tok.check_database(False) is not None
 
 
 
     assert tok.check_database(False) is not None
 
 

-def test_update_statistics_reverse_only(word_table, tokenizer_factory):
+def test_update_statistics_reverse_only(word_table, tokenizer_factory, test_config):

     tok = tokenizer_factory()
     tok = tokenizer_factory()

-    tok.update_statistics()
+    tok.update_statistics(test_config)

 
 
 
 

-def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory):
+def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory, test_config):

     word_table.add_full_word(1000, 'hello')
     table_factory('search_name',
                   'place_id BIGINT, name_vector INT[]',
                   [(12, [1000])])
     tok = tokenizer_factory()
 
     word_table.add_full_word(1000, 'hello')
     table_factory('search_name',
                   'place_id BIGINT, name_vector INT[]',
                   [(12, [1000])])
     tok = tokenizer_factory()
 

-    tok.update_statistics()
+    tok.update_statistics(test_config)

 
     assert temp_db_cursor.scalar("""SELECT count(*) FROM word
                                     WHERE word_token like ' %' and
                                           search_name_count > 0""") > 0
 
 
 
     assert temp_db_cursor.scalar("""SELECT count(*) FROM word
                                     WHERE word_token like ' %' and
                                           search_name_count > 0""") > 0
 
 

+def test_update_word_tokens(tokenizer_factory):
+    tok = tokenizer_factory()
+
+    # This is a noop and should just pass.
+    tok.update_word_tokens()
+
+

 def test_normalize(analyzer):
     assert analyzer.normalize('TEsT') == 'test'
 
 def test_normalize(analyzer):
     assert analyzer.normalize('TEsT') == 'test'
 


@@ -536,7 +549,7 @@ class TestPlaceAddress:
     def test_process_place_street_empty(self):
         info = self.process_address(street='🜵')
 
     def test_process_place_street_empty(self):
         info = self.process_address(street='🜵')
 

-        assert 'street' not in info
+        assert info['street'] == '{}'

 
 
     def test_process_place_place(self):
 
 
     def test_process_place_place(self):