enable CI tests for Ubuntu 24

[nominatim.git] / test / python / tokenizer / test_icu.py

diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py
index aa1afe160ca9010630b7b36502d14b173a453003..357b7d4ae4cde571315c2a7d8109595911e343f4 100644 (file)
--- a/test/python/tokenizer/test_icu.py
+++ b/test/python/tokenizer/test_icu.py
@@ -1,8 +1,8 @@
-# SPDX-License-Identifier: GPL-2.0-only
+# SPDX-License-Identifier: GPL-3.0-or-later

 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #

-# Copyright (C) 2022 by the Nominatim developer community.
+# Copyright (C) 2024 by the Nominatim developer community.

 # For a full list of authors see the git log.
 """
 Tests for ICU tokenizer.
 # For a full list of authors see the git log.
 """
 Tests for ICU tokenizer.


@@ -12,11 +12,11 @@ import itertools
 
 import pytest
 
 
 import pytest
 

-from nominatim.tokenizer import icu_tokenizer
-import nominatim.tokenizer.icu_rule_loader
-from nominatim.db import properties
-from nominatim.db.sql_preprocessor import SQLPreprocessor
-from nominatim.data.place_info import PlaceInfo
+from nominatim_db.tokenizer import icu_tokenizer
+import nominatim_db.tokenizer.icu_rule_loader
+from nominatim_db.db import properties
+from nominatim_db.db.sql_preprocessor import SQLPreprocessor
+from nominatim_db.data.place_info import PlaceInfo

 
 from mock_icu_word_table import MockIcuWordTable
 
 
 from mock_icu_word_table import MockIcuWordTable
 


@@ -83,7 +83,7 @@ def analyzer(tokenizer_factory, test_config, monkeypatch,
             cfgstr['token-analysis'].append({'id': '@postcode',
                                              'analyzer': 'postcodes'})
         (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr))
             cfgstr['token-analysis'].append({'id': '@postcode',
                                              'analyzer': 'postcodes'})
         (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr))

-        tok.loader = nominatim.tokenizer.icu_rule_loader.ICURuleLoader(test_config)
+        tok.loader = nominatim_db.tokenizer.icu_rule_loader.ICURuleLoader(test_config)

 
         return tok.name_analyzer()
 
 
         return tok.name_analyzer()
 


@@ -157,7 +157,7 @@ def test_init_new(tokenizer_factory, test_config, db_prop):
     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 
     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 

-    assert db_prop(nominatim.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES) \
+    assert db_prop(nominatim_db.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES) \

             .startswith(':: lower ();')
 
 
             .startswith(':: lower ();')
 
 


@@ -227,16 +227,20 @@ def test_update_statistics_reverse_only(word_table, tokenizer_factory, test_conf
 def test_update_statistics(word_table, table_factory, temp_db_cursor,
                            tokenizer_factory, test_config):
     word_table.add_full_word(1000, 'hello')
 def test_update_statistics(word_table, table_factory, temp_db_cursor,
                            tokenizer_factory, test_config):
     word_table.add_full_word(1000, 'hello')

+    word_table.add_full_word(1001, 'bye')

     table_factory('search_name',
     table_factory('search_name',

-                  'place_id BIGINT, name_vector INT[]',
-                  [(12, [1000])])
+                  'place_id BIGINT, name_vector INT[], nameaddress_vector INT[]',
+                  [(12, [1000], [1001])])

     tok = tokenizer_factory()
 
     tok.update_statistics(test_config)
 
     assert temp_db_cursor.scalar("""SELECT count(*) FROM word
     tok = tokenizer_factory()
 
     tok.update_statistics(test_config)
 
     assert temp_db_cursor.scalar("""SELECT count(*) FROM word

-                                    WHERE type = 'W' and
-                                          (info->>'count')::int > 0""") > 0
+                                    WHERE type = 'W' and word_id = 1000 and
+                                          (info->>'count')::int > 0""") == 1
+    assert temp_db_cursor.scalar("""SELECT count(*) FROM word
+                                    WHERE type = 'W' and word_id = 1001 and
+                                          (info->>'addr_count')::int > 0""") == 1

 
 
 def test_normalize_postcode(analyzer):
 
 
 def test_normalize_postcode(analyzer):


@@ -550,7 +554,7 @@ class TestPlaceAddress:
     def test_process_place_place(self):
         info = self.process_address(place='Honu Lulu')
 
     def test_process_place_place(self):
         info = self.process_address(place='Honu Lulu')
 

-        assert eval(info['place']) == self.name_token_set('HONU', 'LULU')
+        assert eval(info['place']) == self.name_token_set('HONU', 'LULU', '#HONU LULU')

 
 
     def test_process_place_place_extra(self):
 
 
     def test_process_place_place_extra(self):


@@ -570,8 +574,8 @@ class TestPlaceAddress:
                                     suburb='Zwickau', street='Hauptstr',
                                     full='right behind the church')
 
                                     suburb='Zwickau', street='Hauptstr',
                                     full='right behind the church')
 

-        city = self.name_token_set('ZWICKAU')
-        state = self.name_token_set('SACHSEN')
+        city = self.name_token_set('ZWICKAU', '#ZWICKAU')
+        state = self.name_token_set('SACHSEN', '#SACHSEN')

 
         result = {k: eval(v) for k,v in info['addr'].items()}
 
 
         result = {k: eval(v) for k,v in info['addr'].items()}
 


@@ -583,7 +587,7 @@ class TestPlaceAddress:
 
         result = {k: eval(v) for k,v in info['addr'].items()}
 
 
         result = {k: eval(v) for k,v in info['addr'].items()}
 

-        assert result == {'city': self.name_token_set('Bruxelles')}
+        assert result == {'city': self.name_token_set('Bruxelles', '#Bruxelles')}

 
 
     def test_process_place_address_terms_empty(self):
 
 
     def test_process_place_address_terms_empty(self):