nominatim/tokenizer/token_analysis/generic.py

   1 # SPDX-License-Identifier: GPL-2.0-only
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2022 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Generic processor for names that creates abbreviation variants.
   9 """
  10 import itertools
  11
  12 import datrie
  13
  14 from nominatim.tokenizer.token_analysis.config_variants import get_variant_config
  15
  16 ### Configuration section
  17
  18 def configure(rules, normalization_rules):
  19     """ Extract and preprocess the configuration for this module.
  20     """
  21     config = {}
  22
  23     config['replacements'], config['chars'] = get_variant_config(rules.get('variants'),
  24                                                                  normalization_rules)
  25     config['variant_only'] = rules.get('mode', '') == 'variant-only'
  26
  27     return config
  28
  29
  30 ### Analysis section
  31
  32 def create(transliterator, config):
  33     """ Create a new token analysis instance for this module.
  34     """
  35     return GenericTokenAnalysis(transliterator, config)
  36
  37
  38 class GenericTokenAnalysis:
  39     """ Collects the different transformation rules for normalisation of names
  40         and provides the functions to apply the transformations.
  41     """
  42
  43     def __init__(self, to_ascii, config):
  44         self.to_ascii = to_ascii
  45         self.variant_only = config['variant_only']
  46
  47         # Set up datrie
  48         if config['replacements']:
  49             self.replacements = datrie.Trie(config['chars'])
  50             for src, repllist in config['replacements']:
  51                 self.replacements[src] = repllist
  52         else:
  53             self.replacements = None
  54
  55
  56     def get_variants_ascii(self, norm_name):
  57         """ Compute the spelling variants for the given normalized name
  58             and transliterate the result.
  59         """
  60         results = set()
  61         for variant in self._generate_word_variants(norm_name):
  62             if not self.variant_only or variant.strip() != norm_name:
  63                 trans_name = self.to_ascii.transliterate(variant).strip()
  64                 if trans_name:
  65                     results.add(trans_name)
  66
  67         return list(results)
  68
  69
  70     def _generate_word_variants(self, norm_name):
  71         baseform = '^ ' + norm_name + ' ^'
  72         baselen = len(baseform)
  73         partials = ['']
  74
  75         startpos = 0
  76         if self.replacements is not None:
  77             pos = 0
  78             force_space = False
  79             while pos < baselen:
  80                 full, repl = self.replacements.longest_prefix_item(baseform[pos:],
  81                                                                    (None, None))
  82                 if full is not None:
  83                     done = baseform[startpos:pos]
  84                     partials = [v + done + r
  85                                 for v, r in itertools.product(partials, repl)
  86                                 if not force_space or r.startswith(' ')]
  87                     if len(partials) > 128:
  88                         # If too many variants are produced, they are unlikely
  89                         # to be helpful. Only use the original term.
  90                         startpos = 0
  91                         break
  92                     startpos = pos + len(full)
  93                     if full[-1] == ' ':
  94                         startpos -= 1
  95                         force_space = True
  96                     pos = startpos
  97                 else:
  98                     pos += 1
  99                     force_space = False
 100
 101         # No variants detected? Fast return.
 102         if startpos == 0:
 103             return (norm_name, )
 104
 105         if startpos < baselen:
 106             return (part[1:] + baseform[startpos:-1] for part in partials)
 107
 108         return (part[1:-1] for part in partials)