minimum counts for tokens should always be 1

[nominatim.git] / nominatim / tokenizer / token_analysis / generic.py

diff --git a/nominatim/tokenizer/token_analysis/generic.py b/nominatim/tokenizer/token_analysis/generic.py
index 1e7b75a1218ba8330a99d17d6681fa3c8c641b82..1ed9bf4d383107e0c00a071d3f768057499f432e 100644 (file)
--- a/nominatim/tokenizer/token_analysis/generic.py
+++ b/nominatim/tokenizer/token_analysis/generic.py
@@ -7,35 +7,53 @@
 """
 Generic processor for names that creates abbreviation variants.
 """
 """
 Generic processor for names that creates abbreviation variants.
 """

+from typing import Mapping, Dict, Any, Iterable, Iterator, Optional, List, cast

 import itertools
 
 import datrie
 
 from nominatim.errors import UsageError
 import itertools
 
 import datrie
 
 from nominatim.errors import UsageError

+from nominatim.data.place_name import PlaceName

 from nominatim.tokenizer.token_analysis.config_variants import get_variant_config
 from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantGenerator
 
 ### Configuration section
 
 from nominatim.tokenizer.token_analysis.config_variants import get_variant_config
 from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantGenerator
 
 ### Configuration section
 

-def configure(rules, normalization_rules):
+def configure(rules: Mapping[str, Any], normalizer: Any, _: Any) -> Dict[str, Any]:

     """ Extract and preprocess the configuration for this module.
     """
     """ Extract and preprocess the configuration for this module.
     """

-    config = {}
+    config: Dict[str, Any] = {}

 
     config['replacements'], config['chars'] = get_variant_config(rules.get('variants'),
 
     config['replacements'], config['chars'] = get_variant_config(rules.get('variants'),

-                                                                 normalization_rules)
+                                                                 normalizer)

     config['variant_only'] = rules.get('mode', '') == 'variant-only'
     config['variant_only'] = rules.get('mode', '') == 'variant-only'

-    config['mutations'] = rules.get('mutations', [])
+
+    # parse mutation rules
+    config['mutations'] = []
+    for rule in rules.get('mutations', []):
+        if 'pattern' not in rule:
+            raise UsageError("Missing field 'pattern' in mutation configuration.")
+        if not isinstance(rule['pattern'], str):
+            raise UsageError("Field 'pattern' in mutation configuration "
+                             "must be a simple text field.")
+        if 'replacements' not in rule:
+            raise UsageError("Missing field 'replacements' in mutation configuration.")
+        if not isinstance(rule['replacements'], list):
+            raise UsageError("Field 'replacements' in mutation configuration "
+                             "must be a list of texts.")
+
+        config['mutations'].append((rule['pattern'], rule['replacements']))

 
     return config
 
 
 ### Analysis section
 
 
     return config
 
 
 ### Analysis section
 

-def create(transliterator, config):
+def create(normalizer: Any, transliterator: Any,
+           config: Mapping[str, Any]) -> 'GenericTokenAnalysis':

     """ Create a new token analysis instance for this module.
     """
     """ Create a new token analysis instance for this module.
     """

-    return GenericTokenAnalysis(transliterator, config)
+    return GenericTokenAnalysis(normalizer, transliterator, config)

 
 
 class GenericTokenAnalysis:
 
 
 class GenericTokenAnalysis:


@@ -43,7 +61,8 @@ class GenericTokenAnalysis:
         and provides the functions to apply the transformations.
     """
 
         and provides the functions to apply the transformations.
     """
 

-    def __init__(self, to_ascii, config):
+    def __init__(self, norm: Any, to_ascii: Any, config: Mapping[str, Any]) -> None:
+        self.norm = norm

         self.to_ascii = to_ascii
         self.variant_only = config['variant_only']
 
         self.to_ascii = to_ascii
         self.variant_only = config['variant_only']
 


@@ -56,24 +75,17 @@ class GenericTokenAnalysis:
             self.replacements = None
 
         # set up mutation rules
             self.replacements = None
 
         # set up mutation rules

-        self.mutations = []
-        for cfg in config['mutations']:
-            if 'pattern' not in cfg:
-                raise UsageError("Missing field 'pattern' in mutation configuration.")
-            if not isinstance(cfg['pattern'], str):
-                raise UsageError("Field 'pattern' in mutation configuration "
-                                 "must be a simple text field.")
-            if 'replacements' not in cfg:
-                raise UsageError("Missing field 'replacements' in mutation configuration.")
-            if not isinstance(cfg['replacements'], list):
-                raise UsageError("Field 'replacements' in mutation configuration "
-                                 "must be a list of texts.")
-
-            self.mutations.append(MutationVariantGenerator(cfg['pattern'],
-                                                           cfg['replacements']))
-
-
-    def get_variants_ascii(self, norm_name):
+        self.mutations = [MutationVariantGenerator(*cfg) for cfg in config['mutations']]
+
+
+    def get_canonical_id(self, name: PlaceName) -> str:
+        """ Return the normalized form of the name. This is the standard form
+            from which possible variants for the name can be derived.
+        """
+        return cast(str, self.norm.transliterate(name.name)).strip()
+
+
+    def compute_variants(self, norm_name: str) -> List[str]:

         """ Compute the spelling variants for the given normalized name
             and transliterate the result.
         """
         """ Compute the spelling variants for the given normalized name
             and transliterate the result.
         """


@@ -85,7 +97,8 @@ class GenericTokenAnalysis:
         return [name for name in self._transliterate_unique_list(norm_name, variants) if name]
 
 
         return [name for name in self._transliterate_unique_list(norm_name, variants) if name]
 
 

-    def _transliterate_unique_list(self, norm_name, iterable):
+    def _transliterate_unique_list(self, norm_name: str,
+                                   iterable: Iterable[str]) -> Iterator[Optional[str]]:

         seen = set()
         if self.variant_only:
             seen.add(norm_name)
         seen = set()
         if self.variant_only:
             seen.add(norm_name)


@@ -96,7 +109,7 @@ class GenericTokenAnalysis:
                 yield self.to_ascii.transliterate(variant).strip()
 
 
                 yield self.to_ascii.transliterate(variant).strip()
 
 

-    def _generate_word_variants(self, norm_name):
+    def _generate_word_variants(self, norm_name: str) -> Iterable[str]:

         baseform = '^ ' + norm_name + ' ^'
         baselen = len(baseform)
         partials = ['']
         baseform = '^ ' + norm_name + ' ^'
         baselen = len(baseform)
         partials = ['']