Merge remote-tracking branch 'upstream/master'

[nominatim.git] / nominatim / tokenizer / icu_rule_loader.py

diff --git a/nominatim/tokenizer/icu_rule_loader.py b/nominatim/tokenizer/icu_rule_loader.py
index cb38cfdfb0da9e4ca3f79f09ab7d6812df4d5778..8a564355e35c03e4ed715363d505224efac1a6cd 100644 (file)
--- a/nominatim/tokenizer/icu_rule_loader.py
+++ b/nominatim/tokenizer/icu_rule_loader.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Helper class to create ICU rules from a configuration file.
 """
 """
 Helper class to create ICU rules from a configuration file.
 """


@@ -10,6 +16,8 @@ from nominatim.config import flatten_config_list
 from nominatim.db.properties import set_property, get_property
 from nominatim.errors import UsageError
 from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
 from nominatim.db.properties import set_property, get_property
 from nominatim.errors import UsageError
 from nominatim.tokenizer.place_sanitizer import PlaceSanitizer

+from nominatim.tokenizer.icu_token_analysis import ICUTokenAnalysis
+import nominatim.tools.country_info

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 


@@ -37,6 +45,9 @@ class ICURuleLoader:
         rules = config.load_sub_configuration('icu_tokenizer.yaml',
                                               config='TOKENIZER_CONFIG')
 
         rules = config.load_sub_configuration('icu_tokenizer.yaml',
                                               config='TOKENIZER_CONFIG')
 

+        # Make sure country information is available to analyzers and sanitizers.
+        nominatim.tools.country_info.setup_country_config(config)
+

         self.normalization_rules = self._cfg_to_icu_rules(rules, 'normalization')
         self.transliteration_rules = self._cfg_to_icu_rules(rules, 'transliteration')
         self.analysis_rules = _get_section(rules, 'token-analysis')
         self.normalization_rules = self._cfg_to_icu_rules(rules, 'normalization')
         self.transliteration_rules = self._cfg_to_icu_rules(rules, 'transliteration')
         self.analysis_rules = _get_section(rules, 'token-analysis')


@@ -74,8 +85,8 @@ class ICURuleLoader:
     def make_token_analysis(self):
         """ Create a token analyser from the reviouly loaded rules.
         """
     def make_token_analysis(self):
         """ Create a token analyser from the reviouly loaded rules.
         """

-        return self.analysis[None].create(self.normalization_rules,
-                                          self.transliteration_rules)
+        return ICUTokenAnalysis(self.normalization_rules,
+                                self.transliteration_rules, self.analysis)

 
 
     def get_search_rules(self):
 
 
     def get_search_rules(self):


@@ -119,7 +130,7 @@ class ICURuleLoader:
                 else:
                     LOG.fatal("ICU tokenizer configuration has two token "
                               "analyzers with id '%s'.", name)
                 else:
                     LOG.fatal("ICU tokenizer configuration has two token "
                               "analyzers with id '%s'.", name)

-                UsageError("Syntax error in ICU tokenizer config.")
+                raise UsageError("Syntax error in ICU tokenizer config.")

             self.analysis[name] = TokenAnalyzerRule(section, self.normalization_rules)
 
 
             self.analysis[name] = TokenAnalyzerRule(section, self.normalization_rules)
 
 


@@ -149,15 +160,7 @@ class TokenAnalyzerRule:
         module_name = 'nominatim.tokenizer.token_analysis.' \
                       + _get_section(rules, 'analyzer').replace('-', '_')
         analysis_mod = importlib.import_module(module_name)
         module_name = 'nominatim.tokenizer.token_analysis.' \
                       + _get_section(rules, 'analyzer').replace('-', '_')
         analysis_mod = importlib.import_module(module_name)

-        self._mod_create = analysis_mod.create
+        self.create = analysis_mod.create

 
         # Load the configuration.
         self.config = analysis_mod.configure(rules, normalization_rules)
 
         # Load the configuration.
         self.config = analysis_mod.configure(rules, normalization_rules)

-
-
-    def create(self, normalization_rules, transliteration_rules):
-        """ Create an analyzer from the given rules.
-        """
-        return self._mod_create(normalization_rules,
-                                transliteration_rules,
-                                self.config)