use word_token length when penalizing against postcodes

[nominatim.git] / test / python / tokenizer / test_icu_rule_loader.py

diff --git a/test/python/tokenizer/test_icu_rule_loader.py b/test/python/tokenizer/test_icu_rule_loader.py
index a3fae75815d4a1d0dad629b00a547c7d77efc1bb..f26b84c26fbac32ccd02b491b03c6c1e2a86a865 100644 (file)
--- a/test/python/tokenizer/test_icu_rule_loader.py
+++ b/test/python/tokenizer/test_icu_rule_loader.py
@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #

-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.

 # For a full list of authors see the git log.
 """
 Tests for converting a config file to ICU rules.
 # For a full list of authors see the git log.
 """
 Tests for converting a config file to ICU rules.


@@ -19,17 +19,16 @@ from icu import Transliterator
 
 CONFIG_SECTIONS = ('normalization', 'transliteration', 'token-analysis')
 
 
 CONFIG_SECTIONS = ('normalization', 'transliteration', 'token-analysis')
 

+

 class TestIcuRuleLoader:
 
     @pytest.fixture(autouse=True)
     def init_env(self, project_env):
         self.project_env = project_env
 
 class TestIcuRuleLoader:
 
     @pytest.fixture(autouse=True)
     def init_env(self, project_env):
         self.project_env = project_env
 

-

     def write_config(self, content):
         (self.project_env.project_dir / 'icu_tokenizer.yaml').write_text(dedent(content))
 
     def write_config(self, content):
         (self.project_env.project_dir / 'icu_tokenizer.yaml').write_text(dedent(content))
 

-

     def config_rules(self, *variants):
         content = dedent("""\
         normalization:
     def config_rules(self, *variants):
         content = dedent("""\
         normalization:


@@ -49,14 +48,12 @@ class TestIcuRuleLoader:
         content += '\n'.join(("             - " + s for s in variants)) + '\n'
         self.write_config(content)
 
         content += '\n'.join(("             - " + s for s in variants)) + '\n'
         self.write_config(content)
 

-

     def get_replacements(self, *variants):
         self.config_rules(*variants)
         loader = ICURuleLoader(self.project_env)
         rules = loader.analysis[None].config['replacements']
 
     def get_replacements(self, *variants):
         self.config_rules(*variants)
         loader = ICURuleLoader(self.project_env)
         rules = loader.analysis[None].config['replacements']
 

-        return sorted((k, sorted(v)) for k,v in rules)
-
+        return sorted((k, sorted(v)) for k, v in rules)

 
     def test_empty_rule_set(self):
         self.write_config("""\
 
     def test_empty_rule_set(self):
         self.write_config("""\


@@ -72,16 +69,14 @@ class TestIcuRuleLoader:
         assert rules.get_normalization_rules() == ''
         assert rules.get_transliteration_rules() == ''
 
         assert rules.get_normalization_rules() == ''
         assert rules.get_transliteration_rules() == ''
 

-

     @pytest.mark.parametrize("section", CONFIG_SECTIONS)
     def test_missing_section(self, section):
     @pytest.mark.parametrize("section", CONFIG_SECTIONS)
     def test_missing_section(self, section):

-        rule_cfg = { s: [] for s in CONFIG_SECTIONS if s != section}
+        rule_cfg = {s: [] for s in CONFIG_SECTIONS if s != section}

         self.write_config(yaml.dump(rule_cfg))
 
         with pytest.raises(UsageError):
             ICURuleLoader(self.project_env)
 
         self.write_config(yaml.dump(rule_cfg))
 
         with pytest.raises(UsageError):
             ICURuleLoader(self.project_env)
 

-

     def test_get_search_rules(self):
         self.config_rules()
         loader = ICURuleLoader(self.project_env)
     def test_get_search_rules(self):
         self.config_rules()
         loader = ICURuleLoader(self.project_env)


@@ -97,7 +92,6 @@ class TestIcuRuleLoader:
         assert trans.transliterate(" Αθήνα ") == " athēna "
         assert trans.transliterate(" проспект ") == " prospekt "
 
         assert trans.transliterate(" Αθήνα ") == " athēna "
         assert trans.transliterate(" проспект ") == " prospekt "
 

-

     def test_get_normalization_rules(self):
         self.config_rules()
         loader = ICURuleLoader(self.project_env)
     def test_get_normalization_rules(self):
         self.config_rules()
         loader = ICURuleLoader(self.project_env)


@@ -106,7 +100,6 @@ class TestIcuRuleLoader:
 
         assert trans.transliterate(" проспект-Prospekt ") == " проспект prospekt "
 
 
         assert trans.transliterate(" проспект-Prospekt ") == " проспект prospekt "
 

-

     def test_get_transliteration_rules(self):
         self.config_rules()
         loader = ICURuleLoader(self.project_env)
     def test_get_transliteration_rules(self):
         self.config_rules()
         loader = ICURuleLoader(self.project_env)


@@ -115,7 +108,6 @@ class TestIcuRuleLoader:
 
         assert trans.transliterate(" проспект-Prospekt ") == " prospekt Prospekt "
 
 
         assert trans.transliterate(" проспект-Prospekt ") == " prospekt Prospekt "
 

-

     def test_transliteration_rules_from_file(self):
         self.write_config("""\
             normalization:
     def test_transliteration_rules_from_file(self):
         self.write_config("""\
             normalization:


@@ -135,7 +127,6 @@ class TestIcuRuleLoader:
 
         assert trans.transliterate(" axxt ") == " byt "
 
 
         assert trans.transliterate(" axxt ") == " byt "
 

-

     def test_search_rules(self):
         self.config_rules('~street => s,st', 'master => mstr')
         proc = ICURuleLoader(self.project_env).make_token_analysis()
     def test_search_rules(self):
         self.config_rules('~street => s,st', 'master => mstr')
         proc = ICURuleLoader(self.project_env).make_token_analysis()


@@ -144,7 +135,6 @@ class TestIcuRuleLoader:
         assert proc.search.transliterate('Earnes St').strip() == 'earnes st'
         assert proc.search.transliterate('Nostreet').strip() == 'nostreet'
 
         assert proc.search.transliterate('Earnes St').strip() == 'earnes st'
         assert proc.search.transliterate('Nostreet').strip() == 'nostreet'
 

-

     @pytest.mark.parametrize("variant", ['foo > bar', 'foo -> bar -> bar',
                                          '~foo~ -> bar', 'fo~ o -> bar'])
     def test_invalid_variant_description(self, variant):
     @pytest.mark.parametrize("variant", ['foo > bar', 'foo -> bar -> bar',
                                          '~foo~ -> bar', 'fo~ o -> bar'])
     def test_invalid_variant_description(self, variant):


@@ -157,25 +147,21 @@ class TestIcuRuleLoader:
 
         assert repl == [(' foo ', [' bar', ' foo'])]
 
 
         assert repl == [(' foo ', [' bar', ' foo'])]
 

-

     def test_replace_full(self):
         repl = self.get_replacements("foo => bar")
 
         assert repl == [(' foo ', [' bar'])]
 
     def test_replace_full(self):
         repl = self.get_replacements("foo => bar")
 
         assert repl == [(' foo ', [' bar'])]
 

-

     def test_add_suffix_no_decompose(self):
         repl = self.get_replacements("~berg |-> bg")
 
         assert repl == [(' berg ', [' berg', ' bg']),
                         ('berg ', ['berg', 'bg'])]
 
     def test_add_suffix_no_decompose(self):
         repl = self.get_replacements("~berg |-> bg")
 
         assert repl == [(' berg ', [' berg', ' bg']),
                         ('berg ', ['berg', 'bg'])]
 

-

     def test_replace_suffix_no_decompose(self):
         repl = self.get_replacements("~berg |=> bg")
 
     def test_replace_suffix_no_decompose(self):
         repl = self.get_replacements("~berg |=> bg")
 

-        assert repl == [(' berg ', [' bg']),('berg ', ['bg'])]
-
+        assert repl == [(' berg ', [' bg']), ('berg ', ['bg'])]

 
     def test_add_suffix_decompose(self):
         repl = self.get_replacements("~berg -> bg")
 
     def test_add_suffix_decompose(self):
         repl = self.get_replacements("~berg -> bg")


@@ -183,26 +169,22 @@ class TestIcuRuleLoader:
         assert repl == [(' berg ', [' berg', ' bg', 'berg', 'bg']),
                         ('berg ', [' berg', ' bg', 'berg', 'bg'])]
 
         assert repl == [(' berg ', [' berg', ' bg', 'berg', 'bg']),
                         ('berg ', [' berg', ' bg', 'berg', 'bg'])]
 

-

     def test_replace_suffix_decompose(self):
         repl = self.get_replacements("~berg => bg")
 
         assert repl == [(' berg ', [' bg', 'bg']),
                         ('berg ', [' bg', 'bg'])]
 
     def test_replace_suffix_decompose(self):
         repl = self.get_replacements("~berg => bg")
 
         assert repl == [(' berg ', [' bg', 'bg']),
                         ('berg ', [' bg', 'bg'])]
 

-

     def test_add_prefix_no_compose(self):
         repl = self.get_replacements("hinter~ |-> hnt")
 
         assert repl == [(' hinter', [' hinter', ' hnt']),
                         (' hinter ', [' hinter', ' hnt'])]
 
     def test_add_prefix_no_compose(self):
         repl = self.get_replacements("hinter~ |-> hnt")
 
         assert repl == [(' hinter', [' hinter', ' hnt']),
                         (' hinter ', [' hinter', ' hnt'])]
 

-

     def test_replace_prefix_no_compose(self):
         repl = self.get_replacements("hinter~ |=> hnt")
 
     def test_replace_prefix_no_compose(self):
         repl = self.get_replacements("hinter~ |=> hnt")
 

-        assert repl ==  [(' hinter', [' hnt']), (' hinter ', [' hnt'])]
-
+        assert repl == [(' hinter', [' hnt']), (' hinter ', [' hnt'])]

 
     def test_add_prefix_compose(self):
         repl = self.get_replacements("hinter~-> h")
 
     def test_add_prefix_compose(self):
         repl = self.get_replacements("hinter~-> h")


@@ -210,45 +192,38 @@ class TestIcuRuleLoader:
         assert repl == [(' hinter', [' h', ' h ', ' hinter', ' hinter ']),
                         (' hinter ', [' h', ' h', ' hinter', ' hinter'])]
 
         assert repl == [(' hinter', [' h', ' h ', ' hinter', ' hinter ']),
                         (' hinter ', [' h', ' h', ' hinter', ' hinter'])]
 

-

     def test_replace_prefix_compose(self):
         repl = self.get_replacements("hinter~=> h")
 
         assert repl == [(' hinter', [' h', ' h ']),
                         (' hinter ', [' h', ' h'])]
 
     def test_replace_prefix_compose(self):
         repl = self.get_replacements("hinter~=> h")
 
         assert repl == [(' hinter', [' h', ' h ']),
                         (' hinter ', [' h', ' h'])]
 

-

     def test_add_beginning_only(self):
         repl = self.get_replacements("^Premier -> Pr")
 
         assert repl == [('^ premier ', ['^ pr', '^ premier'])]
 
     def test_add_beginning_only(self):
         repl = self.get_replacements("^Premier -> Pr")
 
         assert repl == [('^ premier ', ['^ pr', '^ premier'])]
 

-

     def test_replace_beginning_only(self):
         repl = self.get_replacements("^Premier => Pr")
 
         assert repl == [('^ premier ', ['^ pr'])]
 
     def test_replace_beginning_only(self):
         repl = self.get_replacements("^Premier => Pr")
 
         assert repl == [('^ premier ', ['^ pr'])]
 

-

     def test_add_final_only(self):
         repl = self.get_replacements("road$ -> rd")
 
         assert repl == [(' road ^', [' rd ^', ' road ^'])]
 
     def test_add_final_only(self):
         repl = self.get_replacements("road$ -> rd")
 
         assert repl == [(' road ^', [' rd ^', ' road ^'])]
 

-

     def test_replace_final_only(self):
         repl = self.get_replacements("road$ => rd")
 
         assert repl == [(' road ^', [' rd ^'])]
 
     def test_replace_final_only(self):
         repl = self.get_replacements("road$ => rd")
 
         assert repl == [(' road ^', [' rd ^'])]
 

-

     def test_decompose_only(self):
         repl = self.get_replacements("~foo -> foo")
 
         assert repl == [(' foo ', [' foo', 'foo']),
                         ('foo ', [' foo', 'foo'])]
 
     def test_decompose_only(self):
         repl = self.get_replacements("~foo -> foo")
 
         assert repl == [(' foo ', [' foo', 'foo']),
                         ('foo ', [' foo', 'foo'])]
 

-

     def test_add_suffix_decompose_end_only(self):
         repl = self.get_replacements("~berg |-> bg", "~berg$ -> bg")
 
     def test_add_suffix_decompose_end_only(self):
         repl = self.get_replacements("~berg |-> bg", "~berg$ -> bg")
 


@@ -257,7 +232,6 @@ class TestIcuRuleLoader:
                         ('berg ', ['berg', 'bg']),
                         ('berg ^', [' berg ^', ' bg ^', 'berg ^', 'bg ^'])]
 
                         ('berg ', ['berg', 'bg']),
                         ('berg ^', [' berg ^', ' bg ^', 'berg ^', 'bg ^'])]
 

-

     def test_replace_suffix_decompose_end_only(self):
         repl = self.get_replacements("~berg |=> bg", "~berg$ => bg")
 
     def test_replace_suffix_decompose_end_only(self):
         repl = self.get_replacements("~berg |=> bg", "~berg$ => bg")
 


@@ -266,7 +240,6 @@ class TestIcuRuleLoader:
                         ('berg ', ['bg']),
                         ('berg ^', [' bg ^', 'bg ^'])]
 
                         ('berg ', ['bg']),
                         ('berg ^', [' bg ^', 'bg ^'])]
 

-

     def test_add_multiple_suffix(self):
         repl = self.get_replacements("~berg,~burg -> bg")
 
     def test_add_multiple_suffix(self):
         repl = self.get_replacements("~berg,~burg -> bg")