minimum counts for tokens should always be 1

[nominatim.git] / nominatim / tokenizer / token_analysis / config_variants.py

diff --git a/nominatim/tokenizer/token_analysis/config_variants.py b/nominatim/tokenizer/token_analysis/config_variants.py
index 59ceeb225ddee449f697ce1205bdd559f74b93cb..1258373eea9230ff3552e243ae726f4c0a4b2b2b 100644 (file)
--- a/nominatim/tokenizer/token_analysis/config_variants.py
+++ b/nominatim/tokenizer/token_analysis/config_variants.py
@@ -7,18 +7,23 @@
 """
 Parser for configuration for variants.
 """
 """
 Parser for configuration for variants.
 """

-from collections import defaultdict, namedtuple
+from typing import Any, Iterator, Tuple, List, Optional, Set, NamedTuple
+from collections import defaultdict

 import itertools
 import re
 
 import itertools
 import re
 

-from icu import Transliterator
-

 from nominatim.config import flatten_config_list
 from nominatim.errors import UsageError
 
 from nominatim.config import flatten_config_list
 from nominatim.errors import UsageError
 

-ICUVariant = namedtuple('ICUVariant', ['source', 'replacement'])
+class ICUVariant(NamedTuple):
+    """ A single replacement rule for variant creation.
+    """
+    source: str
+    replacement: str
+

 
 

-def get_variant_config(rules, normalization_rules):
+def get_variant_config(in_rules: Any,
+                       normalizer: Any) -> Tuple[List[Tuple[str, List[str]]], str]:

     """ Convert the variant definition from the configuration into
         replacement sets.
 
     """ Convert the variant definition from the configuration into
         replacement sets.
 


@@ -26,13 +31,13 @@ def get_variant_config(rules, normalization_rules):
         used in the replacements.
     """
     immediate = defaultdict(list)
         used in the replacements.
     """
     immediate = defaultdict(list)

-    chars = set()
+    chars: Set[str] = set()

 
 

-    if rules:
-        vset = set()
-        rules = flatten_config_list(rules, 'variants')
+    if in_rules:
+        vset: Set[ICUVariant] = set()
+        rules = flatten_config_list(in_rules, 'variants')

 
 

-        vmaker = _VariantMaker(normalization_rules)
+        vmaker = _VariantMaker(normalizer)

 
         for section in rules:
             for rule in (section.get('words') or []):
 
         for section in rules:
             for rule in (section.get('words') or []):


@@ -51,22 +56,21 @@ def get_variant_config(rules, normalization_rules):
 
 
 class _VariantMaker:
 
 
 class _VariantMaker:

-    """ Generater for all necessary ICUVariants from a single variant rule.
+    """ Generator for all necessary ICUVariants from a single variant rule.

 
         All text in rules is normalized to make sure the variants match later.
     """
 
 
         All text in rules is normalized to make sure the variants match later.
     """
 

-    def __init__(self, norm_rules):
-        self.norm = Transliterator.createFromRules("rule_loader_normalization",
-                                                   norm_rules)
+    def __init__(self, normalizer: Any) -> None:
+        self.norm = normalizer

 
 
 
 

-    def compute(self, rule):
+    def compute(self, rule: Any) -> Iterator[ICUVariant]:

         """ Generator for all ICUVariant tuples from a single variant rule.
         """
         parts = re.split(r'(\|)?([=-])>', rule)
         if len(parts) != 4:
         """ Generator for all ICUVariant tuples from a single variant rule.
         """
         parts = re.split(r'(\|)?([=-])>', rule)
         if len(parts) != 4:

-            raise UsageError("Syntax error in variant rule: " + rule)
+            raise UsageError(f"Syntax error in variant rule: {rule}")

 
         decompose = parts[1] is None
         src_terms = [self._parse_variant_word(t) for t in parts[0].split(',')]
 
         decompose = parts[1] is None
         src_terms = [self._parse_variant_word(t) for t in parts[0].split(',')]


@@ -85,11 +89,11 @@ class _VariantMaker:
                     yield ICUVariant(froms, tos)
 
 
                     yield ICUVariant(froms, tos)
 
 

-    def _parse_variant_word(self, name):
+    def _parse_variant_word(self, name: str) -> Optional[Tuple[str, str, str]]:

         name = name.strip()
         match = re.fullmatch(r'([~^]?)([^~$^]*)([~$]?)', name)
         if match is None or (match.group(1) == '~' and match.group(3) == '~'):
         name = name.strip()
         match = re.fullmatch(r'([~^]?)([^~$^]*)([~$]?)', name)
         if match is None or (match.group(1) == '~' and match.group(3) == '~'):

-            raise UsageError("Invalid variant word descriptor '{}'".format(name))
+            raise UsageError(f"Invalid variant word descriptor '{name}'")

         norm_name = self.norm.transliterate(match.group(2)).strip()
         if not norm_name:
             return None
         norm_name = self.norm.transliterate(match.group(2)).strip()
         if not norm_name:
             return None


@@ -102,7 +106,8 @@ _FLAG_MATCH = {'^': '^ ',
                '': ' '}
 
 
                '': ' '}
 
 

-def _create_variants(src, preflag, postflag, repl, decompose):
+def _create_variants(src: str, preflag: str, postflag: str,
+                     repl: str, decompose: bool) -> Iterator[Tuple[str, str]]:

     if preflag == '~':
         postfix = _FLAG_MATCH[postflag]
         # suffix decomposition
     if preflag == '~':
         postfix = _FLAG_MATCH[postflag]
         # suffix decomposition