nominatim/tokenizer/icu_name_processor.py

   1 """
   2 Processor for names that are imported into the database based on the
   3 ICU library.
   4 """
   5 from collections import defaultdict
   6 import itertools
   7
   8 from icu import Transliterator
   9 import datrie
  10
  11
  12 class ICUNameProcessor:
  13     """ Collects the different transformation rules for normalisation of names
  14         and provides the functions to apply the transformations.
  15     """
  16
  17     def __init__(self, norm_rules, trans_rules, replacements):
  18         self.normalizer = Transliterator.createFromRules("icu_normalization",
  19                                                          norm_rules)
  20         self.to_ascii = Transliterator.createFromRules("icu_to_ascii",
  21                                                        trans_rules +
  22                                                        ";[:Space:]+ > ' '")
  23         self.search = Transliterator.createFromRules("icu_search",
  24                                                      norm_rules + trans_rules)
  25
  26         # Intermediate reorder by source. Also compute required character set.
  27         immediate = defaultdict(list)
  28         chars = set()
  29         for variant in replacements:
  30             if variant.source[-1] == ' ' and variant.replacement[-1] == ' ':
  31                 replstr = variant.replacement[:-1]
  32             else:
  33                 replstr = variant.replacement
  34             immediate[variant.source].append(replstr)
  35             chars.update(variant.source)
  36         # Then copy to datrie
  37         self.replacements = datrie.Trie(''.join(chars))
  38         for src, repllist in immediate.items():
  39             self.replacements[src] = repllist
  40
  41
  42     def get_normalized(self, name):
  43         """ Normalize the given name, i.e. remove all elements not relevant
  44             for search.
  45         """
  46         return self.normalizer.transliterate(name).strip()
  47
  48     def get_variants_ascii(self, norm_name):
  49         """ Compute the spelling variants for the given normalized name
  50             and transliterate the result.
  51         """
  52         baseform = '^ ' + norm_name + ' ^'
  53         partials = ['']
  54
  55         startpos = 0
  56         pos = 0
  57         force_space = False
  58         while pos < len(baseform):
  59             full, repl = self.replacements.longest_prefix_item(baseform[pos:],
  60                                                                (None, None))
  61             if full is not None:
  62                 done = baseform[startpos:pos]
  63                 partials = [v + done + r
  64                             for v, r in itertools.product(partials, repl)
  65                             if not force_space or r.startswith(' ')]
  66                 if len(partials) > 128:
  67                     # If too many variants are produced, they are unlikely
  68                     # to be helpful. Only use the original term.
  69                     startpos = 0
  70                     break
  71                 startpos = pos + len(full)
  72                 if full[-1] == ' ':
  73                     startpos -= 1
  74                     force_space = True
  75                 pos = startpos
  76             else:
  77                 pos += 1
  78                 force_space = False
  79
  80         # No variants detected? Fast return.
  81         if startpos == 0:
  82             trans_name = self.to_ascii.transliterate(norm_name).strip()
  83             return [trans_name] if trans_name else []
  84
  85         return self._compute_result_set(partials, baseform[startpos:])
  86
  87
  88     def _compute_result_set(self, partials, prefix):
  89         results = set()
  90
  91         for variant in partials:
  92             vname = variant + prefix
  93             trans_name = self.to_ascii.transliterate(vname[1:-1]).strip()
  94             if trans_name:
  95                 results.add(trans_name)
  96
  97         return list(results)
  98
  99
 100     def get_search_normalized(self, name):
 101         """ Return the normalized version of the name (including transliteration)
 102             to be applied at search time.
 103         """
 104         return self.search.transliterate(' ' + name + ' ').strip()