Merge pull request #2784 from lonvia/doscs-customizing-icu-tokenizer

[nominatim.git] / nominatim / tokenizer / token_analysis / generic.py

diff --git a/nominatim/tokenizer/token_analysis/generic.py b/nominatim/tokenizer/token_analysis/generic.py
index 3de915ba5254e1859976dd7e9842247df5a58b98..1ed9bf4d383107e0c00a071d3f768057499f432e 100644 (file)
--- a/nominatim/tokenizer/token_analysis/generic.py
+++ b/nominatim/tokenizer/token_analysis/generic.py
@@ -7,23 +7,25 @@
 """
 Generic processor for names that creates abbreviation variants.
 """
 """
 Generic processor for names that creates abbreviation variants.
 """

+from typing import Mapping, Dict, Any, Iterable, Iterator, Optional, List, cast

 import itertools
 
 import datrie
 
 from nominatim.errors import UsageError
 import itertools
 
 import datrie
 
 from nominatim.errors import UsageError

+from nominatim.data.place_name import PlaceName

 from nominatim.tokenizer.token_analysis.config_variants import get_variant_config
 from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantGenerator
 
 ### Configuration section
 
 from nominatim.tokenizer.token_analysis.config_variants import get_variant_config
 from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantGenerator
 
 ### Configuration section
 

-def configure(rules, normalization_rules):
+def configure(rules: Mapping[str, Any], normalizer: Any, _: Any) -> Dict[str, Any]:

     """ Extract and preprocess the configuration for this module.
     """
     """ Extract and preprocess the configuration for this module.
     """

-    config = {}
+    config: Dict[str, Any] = {}

 
     config['replacements'], config['chars'] = get_variant_config(rules.get('variants'),
 
     config['replacements'], config['chars'] = get_variant_config(rules.get('variants'),

-                                                                 normalization_rules)
+                                                                 normalizer)

     config['variant_only'] = rules.get('mode', '') == 'variant-only'
 
     # parse mutation rules
     config['variant_only'] = rules.get('mode', '') == 'variant-only'
 
     # parse mutation rules


@@ -47,7 +49,8 @@ def configure(rules, normalization_rules):
 
 ### Analysis section
 
 
 ### Analysis section
 

-def create(normalizer, transliterator, config):
+def create(normalizer: Any, transliterator: Any,
+           config: Mapping[str, Any]) -> 'GenericTokenAnalysis':

     """ Create a new token analysis instance for this module.
     """
     return GenericTokenAnalysis(normalizer, transliterator, config)
     """ Create a new token analysis instance for this module.
     """
     return GenericTokenAnalysis(normalizer, transliterator, config)


@@ -58,7 +61,7 @@ class GenericTokenAnalysis:
         and provides the functions to apply the transformations.
     """
 
         and provides the functions to apply the transformations.
     """
 

-    def __init__(self, norm, to_ascii, config):
+    def __init__(self, norm: Any, to_ascii: Any, config: Mapping[str, Any]) -> None:

         self.norm = norm
         self.to_ascii = to_ascii
         self.variant_only = config['variant_only']
         self.norm = norm
         self.to_ascii = to_ascii
         self.variant_only = config['variant_only']


@@ -75,14 +78,14 @@ class GenericTokenAnalysis:
         self.mutations = [MutationVariantGenerator(*cfg) for cfg in config['mutations']]
 
 
         self.mutations = [MutationVariantGenerator(*cfg) for cfg in config['mutations']]
 
 

-    def normalize(self, name):
+    def get_canonical_id(self, name: PlaceName) -> str:

         """ Return the normalized form of the name. This is the standard form
             from which possible variants for the name can be derived.
         """
         """ Return the normalized form of the name. This is the standard form
             from which possible variants for the name can be derived.
         """

-        return self.norm.transliterate(name).strip()
+        return cast(str, self.norm.transliterate(name.name)).strip()

 
 
 
 

-    def get_variants_ascii(self, norm_name):
+    def compute_variants(self, norm_name: str) -> List[str]:

         """ Compute the spelling variants for the given normalized name
             and transliterate the result.
         """
         """ Compute the spelling variants for the given normalized name
             and transliterate the result.
         """


@@ -94,7 +97,8 @@ class GenericTokenAnalysis:
         return [name for name in self._transliterate_unique_list(norm_name, variants) if name]
 
 
         return [name for name in self._transliterate_unique_list(norm_name, variants) if name]
 
 

-    def _transliterate_unique_list(self, norm_name, iterable):
+    def _transliterate_unique_list(self, norm_name: str,
+                                   iterable: Iterable[str]) -> Iterator[Optional[str]]:

         seen = set()
         if self.variant_only:
             seen.add(norm_name)
         seen = set()
         if self.variant_only:
             seen.add(norm_name)


@@ -105,7 +109,7 @@ class GenericTokenAnalysis:
                 yield self.to_ascii.transliterate(variant).strip()
 
 
                 yield self.to_ascii.transliterate(variant).strip()
 
 

-    def _generate_word_variants(self, norm_name):
+    def _generate_word_variants(self, norm_name: str) -> Iterable[str]:

         baseform = '^ ' + norm_name + ' ^'
         baselen = len(baseform)
         partials = ['']
         baseform = '^ ' + norm_name + ' ^'
         baselen = len(baseform)
         partials = ['']