keep break indicators [:-] during normalisation

[nominatim.git] / src / nominatim_db / tokenizer / icu_token_analysis.py

diff --git a/src/nominatim_db/tokenizer/icu_token_analysis.py b/src/nominatim_db/tokenizer/icu_token_analysis.py
index fe6704d4249158526ababae3b4910da1f013cd30..c1ba106c48775498ad7b2596ef460be8f85bc299 100644 (file)
--- a/src/nominatim_db/tokenizer/icu_token_analysis.py
+++ b/src/nominatim_db/tokenizer/icu_token_analysis.py
@@ -14,8 +14,9 @@ from icu import Transliterator
 from .token_analysis.base import Analyzer
 
 if TYPE_CHECKING:
-    from typing import Any
-    from .icu_rule_loader import TokenAnalyzerRule # pylint: disable=cyclic-import
+    from typing import Any  # noqa
+    from .icu_rule_loader import TokenAnalyzerRule
+
 
 class ICUTokenAnalysis:
     """ Container class collecting the transliterators and token analysis
@@ -24,6 +25,8 @@ class ICUTokenAnalysis:
 
     def __init__(self, norm_rules: str, trans_rules: str,
                  analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']):
+        # additional break signs are not relevant during name analysis
+        norm_rules += ";[[:Space:][-:]]+ > ' ';"
         self.normalizer = Transliterator.createFromRules("icu_normalization",
                                                          norm_rules)
         trans_rules += ";[:Space:]+ > ' '"
@@ -35,7 +38,6 @@ class ICUTokenAnalysis:
         self.analysis = {name: arules.create(self.normalizer, self.to_ascii)
                          for name, arules in analysis_rules.items()}
 
-
     def get_analyzer(self, name: Optional[str]) -> Analyzer:
         """ Return the given named analyzer. If no analyzer with that
             name exists, return the default analyzer.