X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/92f6ec2328606666032f7e9adc0c3cdcd7c76804..33b611e24313db83b7a5e175a4e11c5f66c4d25d:/nominatim/tokenizer/token_analysis/generic.py diff --git a/nominatim/tokenizer/token_analysis/generic.py b/nominatim/tokenizer/token_analysis/generic.py index 18dd5dfe..4b47889e 100644 --- a/nominatim/tokenizer/token_analysis/generic.py +++ b/nominatim/tokenizer/token_analysis/generic.py @@ -18,7 +18,19 @@ ICUVariant = namedtuple('ICUVariant', ['source', 'replacement']) def configure(rules, normalization_rules): """ Extract and preprocess the configuration for this module. """ - rules = rules.get('variants') + config = {} + + config['replacements'], config['chars'] = _get_variant_config(rules.get('variants'), + normalization_rules) + config['variant_only'] = rules.get('mode', '') == 'variant-only' + + return config + + +def _get_variant_config(rules, normalization_rules): + """ Convert the variant definition from the configuration into + replacement sets. + """ immediate = defaultdict(list) chars = set() @@ -41,8 +53,7 @@ def configure(rules, normalization_rules): immediate[variant.source].append(replstr) chars.update(variant.source) - return {'replacements': list(immediate.items()), - 'chars': ''.join(chars)} + return list(immediate.items()), ''.join(chars) class _VariantMaker: @@ -65,7 +76,7 @@ class _VariantMaker: decompose = parts[1] is None src_terms = [self._parse_variant_word(t) for t in parts[0].split(',')] - repl_terms = (self.norm.transliterate(t.strip()) for t in parts[3].split(',')) + repl_terms = (self.norm.transliterate(t).strip() for t in parts[3].split(',')) # If the source should be kept, add a 1:1 replacement if parts[2] == '-': @@ -85,7 +96,7 @@ class _VariantMaker: match = re.fullmatch(r'([~^]?)([^~$^]*)([~$]?)', name) if match is None or (match.group(1) == '~' and match.group(3) == '~'): raise UsageError("Invalid variant word descriptor '{}'".format(name)) - norm_name = self.norm.transliterate(match.group(2)) + norm_name = self.norm.transliterate(match.group(2)).strip() if not norm_name: return None @@ -131,10 +142,10 @@ def _create_variants(src, preflag, postflag, repl, decompose): ### Analysis section -def create(norm_rules, trans_rules, config): +def create(transliterator, config): """ Create a new token analysis instance for this module. """ - return GenericTokenAnalysis(norm_rules, trans_rules, config) + return GenericTokenAnalysis(transliterator, config) class GenericTokenAnalysis: @@ -142,26 +153,18 @@ class GenericTokenAnalysis: and provides the functions to apply the transformations. """ - def __init__(self, norm_rules, trans_rules, config): - self.normalizer = Transliterator.createFromRules("icu_normalization", - norm_rules) - self.to_ascii = Transliterator.createFromRules("icu_to_ascii", - trans_rules + - ";[:Space:]+ > ' '") - self.search = Transliterator.createFromRules("icu_search", - norm_rules + trans_rules) + def __init__(self, to_ascii, config): + self.to_ascii = to_ascii + self.variant_only = config['variant_only'] # Set up datrie - self.replacements = datrie.Trie(config['chars']) - for src, repllist in config['replacements']: - self.replacements[src] = repllist - + if config['replacements']: + self.replacements = datrie.Trie(config['chars']) + for src, repllist in config['replacements']: + self.replacements[src] = repllist + else: + self.replacements = None - def get_normalized(self, name): - """ Normalize the given name, i.e. remove all elements not relevant - for search. - """ - return self.normalizer.transliterate(name).strip() def get_variants_ascii(self, norm_name): """ Compute the spelling variants for the given normalized name @@ -171,52 +174,51 @@ class GenericTokenAnalysis: partials = [''] startpos = 0 - pos = 0 - force_space = False - while pos < len(baseform): - full, repl = self.replacements.longest_prefix_item(baseform[pos:], - (None, None)) - if full is not None: - done = baseform[startpos:pos] - partials = [v + done + r - for v, r in itertools.product(partials, repl) - if not force_space or r.startswith(' ')] - if len(partials) > 128: - # If too many variants are produced, they are unlikely - # to be helpful. Only use the original term. - startpos = 0 - break - startpos = pos + len(full) - if full[-1] == ' ': - startpos -= 1 - force_space = True - pos = startpos - else: - pos += 1 - force_space = False + if self.replacements is not None: + pos = 0 + force_space = False + while pos < len(baseform): + full, repl = self.replacements.longest_prefix_item(baseform[pos:], + (None, None)) + if full is not None: + done = baseform[startpos:pos] + partials = [v + done + r + for v, r in itertools.product(partials, repl) + if not force_space or r.startswith(' ')] + if len(partials) > 128: + # If too many variants are produced, they are unlikely + # to be helpful. Only use the original term. + startpos = 0 + break + startpos = pos + len(full) + if full[-1] == ' ': + startpos -= 1 + force_space = True + pos = startpos + else: + pos += 1 + force_space = False # No variants detected? Fast return. if startpos == 0: + if self.variant_only: + return [] + trans_name = self.to_ascii.transliterate(norm_name).strip() return [trans_name] if trans_name else [] - return self._compute_result_set(partials, baseform[startpos:]) + return self._compute_result_set(partials, baseform[startpos:], + norm_name if self.variant_only else '') - def _compute_result_set(self, partials, prefix): + def _compute_result_set(self, partials, prefix, exclude): results = set() for variant in partials: - vname = variant + prefix - trans_name = self.to_ascii.transliterate(vname[1:-1]).strip() - if trans_name: - results.add(trans_name) + vname = (variant + prefix)[1:-1].strip() + if vname != exclude: + trans_name = self.to_ascii.transliterate(vname).strip() + if trans_name: + results.add(trans_name) return list(results) - - - def get_search_normalized(self, name): - """ Return the normalized version of the name (including transliteration) - to be applied at search time. - """ - return self.search.transliterate(' ' + name + ' ').strip()