nominatim/tokenizer/icu_token_analysis.py

   1 """
   2 Container class collecting all components required to transform an OSM name
   3 into a Nominatim token.
   4 """
   5
   6 from icu import Transliterator
   7
   8 class ICUTokenAnalysis:
   9     """ Container class collecting the transliterators and token analysis
  10         modules for a single NameAnalyser instance.
  11     """
  12
  13     def __init__(self, norm_rules, trans_rules, analysis_rules):
  14         self.normalizer = Transliterator.createFromRules("icu_normalization",
  15                                                          norm_rules)
  16         trans_rules += ";[:Space:]+ > ' '"
  17         self.to_ascii = Transliterator.createFromRules("icu_to_ascii",
  18                                                        trans_rules)
  19         self.search = Transliterator.createFromRules("icu_search",
  20                                                      norm_rules + trans_rules)
  21
  22         self.analysis = {name: arules.create(self.to_ascii, arules.config)
  23                          for name, arules in analysis_rules.items()}