add type annotations for token analysis

[nominatim.git] / nominatim / tokenizer / token_analysis / housenumbers.py

diff --git a/nominatim/tokenizer/token_analysis/housenumbers.py b/nominatim/tokenizer/token_analysis/housenumbers.py
index 6a838e00d395e4c05f478328aa9b7558d3da84eb..a0f4214d55fee1b6862541409b7e2f6bab434b26 100644 (file)
--- a/nominatim/tokenizer/token_analysis/housenumbers.py
+++ b/nominatim/tokenizer/token_analysis/housenumbers.py
@@ -8,6 +8,7 @@
 Specialized processor for housenumbers. Analyses common housenumber patterns
 and creates variants for them.
 """
 Specialized processor for housenumbers. Analyses common housenumber patterns
 and creates variants for them.
 """

+from typing import Mapping, Any, List, cast

 import re
 
 from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantGenerator
 import re
 
 from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantGenerator


@@ -15,17 +16,18 @@ from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantG
 RE_NON_DIGIT = re.compile('[^0-9]')
 RE_DIGIT_ALPHA = re.compile(r'(\d)\s*([^\d\s␣])')
 RE_ALPHA_DIGIT = re.compile(r'([^\s\d␣])\s*(\d)')
 RE_NON_DIGIT = re.compile('[^0-9]')
 RE_DIGIT_ALPHA = re.compile(r'(\d)\s*([^\d\s␣])')
 RE_ALPHA_DIGIT = re.compile(r'([^\s\d␣])\s*(\d)')

+RE_NAMED_PART = re.compile(r'[a-z]{4}')

 
 ### Configuration section
 
 
 ### Configuration section
 

-def configure(rules, normalization_rules):
+def configure(rules: Mapping[str, Any], normalization_rules: str) -> None: # pylint: disable=W0613

     """ All behaviour is currently hard-coded.
     """
     return None
 
 ### Analysis section
 
     """ All behaviour is currently hard-coded.
     """
     return None
 
 ### Analysis section
 

-def create(normalizer, transliterator, config):
+def create(normalizer: Any, transliterator: Any, config: None) -> 'HousenumberTokenAnalysis': # pylint: disable=W0613

     """ Create a new token analysis instance for this module.
     """
     return HousenumberTokenAnalysis(normalizer, transliterator)
     """ Create a new token analysis instance for this module.
     """
     return HousenumberTokenAnalysis(normalizer, transliterator)


@@ -34,26 +36,32 @@ def create(normalizer, transliterator, config):
 class HousenumberTokenAnalysis:
     """ Detects common housenumber patterns and normalizes them.
     """
 class HousenumberTokenAnalysis:
     """ Detects common housenumber patterns and normalizes them.
     """

-    def __init__(self, norm, trans):
+    def __init__(self, norm: Any, trans: Any) -> None:

         self.norm = norm
         self.trans = trans
 
         self.mutator = MutationVariantGenerator('␣', (' ', ''))
 
         self.norm = norm
         self.trans = trans
 
         self.mutator = MutationVariantGenerator('␣', (' ', ''))
 

-    def normalize(self, name):
+    def normalize(self, name: str) -> str:

         """ Return the normalized form of the housenumber.
         """
         # shortcut for number-only numbers, which make up 90% of the data.
         if RE_NON_DIGIT.search(name) is None:
             return name
 
         """ Return the normalized form of the housenumber.
         """
         # shortcut for number-only numbers, which make up 90% of the data.
         if RE_NON_DIGIT.search(name) is None:
             return name
 

-        norm = self.trans.transliterate(self.norm.transliterate(name))
-        norm = RE_DIGIT_ALPHA.sub(r'\1␣\2', norm)
-        norm = RE_ALPHA_DIGIT.sub(r'\1␣\2', norm)
+        norm = cast(str, self.trans.transliterate(self.norm.transliterate(name)))
+        # If there is a significant non-numeric part, use as is.
+        if RE_NAMED_PART.search(norm) is None:
+            # Otherwise add optional spaces between digits and letters.
+            (norm_opt, cnt1) = RE_DIGIT_ALPHA.subn(r'\1␣\2', norm)
+            (norm_opt, cnt2) = RE_ALPHA_DIGIT.subn(r'\1␣\2', norm_opt)
+            # Avoid creating too many variants per number.
+            if cnt1 + cnt2 <= 4:
+                return norm_opt

 
         return norm
 
 
         return norm
 

-    def get_variants_ascii(self, norm_name):
+    def get_variants_ascii(self, norm_name: str) -> List[str]:

         """ Compute the spelling variants for the given normalized housenumber.
 
             Generates variants for optional spaces (marked with '␣').
         """ Compute the spelling variants for the given normalized housenumber.
 
             Generates variants for optional spaces (marked with '␣').