X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/bb18479d5bfd59fe418f21e79136646133e93567..ca7b46511d41d67e229f758e638367c241815c11:/nominatim/tokenizer/icu_tokenizer.py diff --git a/nominatim/tokenizer/icu_tokenizer.py b/nominatim/tokenizer/icu_tokenizer.py index 5768fd35..e9812ba0 100644 --- a/nominatim/tokenizer/icu_tokenizer.py +++ b/nominatim/tokenizer/icu_tokenizer.py @@ -1,20 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. """ Tokenizer implementing normalisation as used before Nominatim 4 but using libICU instead of the PostgreSQL module. """ -from collections import Counter import itertools import json import logging -import re from textwrap import dedent from nominatim.db.connection import connect -from nominatim.db.properties import set_property, get_property from nominatim.db.utils import CopyBuffer from nominatim.db.sql_preprocessor import SQLPreprocessor +from nominatim.indexer.place_info import PlaceInfo from nominatim.tokenizer.icu_rule_loader import ICURuleLoader -from nominatim.tokenizer.icu_name_processor import ICUNameProcessor, ICUNameProcessorRules from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer DBCFG_TERM_NORMALIZATION = "tokenizer_term_normalization" @@ -36,8 +39,7 @@ class LegacyICUTokenizer(AbstractTokenizer): def __init__(self, dsn, data_dir): self.dsn = dsn self.data_dir = data_dir - self.naming_rules = None - self.term_normalization = None + self.loader = None def init_new_db(self, config, init_db=True): @@ -46,12 +48,9 @@ class LegacyICUTokenizer(AbstractTokenizer): This copies all necessary data in the project directory to make sure the tokenizer remains stable even over updates. """ - loader = ICURuleLoader(config.load_sub_configuration('icu_tokenizer.yaml', - config='TOKENIZER_CONFIG')) - self.naming_rules = ICUNameProcessorRules(loader=loader) - self.term_normalization = config.TERM_NORMALIZATION + self.loader = ICURuleLoader(config) - self._install_php(config.lib_dir.php) + self._install_php(config.lib_dir.php, overwrite=True) self._save_config() if init_db: @@ -59,18 +58,24 @@ class LegacyICUTokenizer(AbstractTokenizer): self._init_db_tables(config) - def init_from_project(self): + def init_from_project(self, config): """ Initialise the tokenizer from the project directory. """ + self.loader = ICURuleLoader(config) + with connect(self.dsn) as conn: - self.naming_rules = ICUNameProcessorRules(conn=conn) - self.term_normalization = get_property(conn, DBCFG_TERM_NORMALIZATION) + self.loader.load_config_from_db(conn) + + self._install_php(config.lib_dir.php, overwrite=False) - def finalize_import(self, _): + def finalize_import(self, config): """ Do any required postprocessing to make the tokenizer data ready for use. """ + with connect(self.dsn) as conn: + sqlp = SQLPreprocessor(conn, config) + sqlp.run_sql_file(conn, 'tokenizer/legacy_tokenizer_indices.sql') def update_sql_functions(self, config): @@ -81,15 +86,74 @@ class LegacyICUTokenizer(AbstractTokenizer): sqlp.run_sql_file(conn, 'tokenizer/icu_tokenizer.sql') - def check_database(self): + def check_database(self, config): """ Check that the tokenizer is set up correctly. """ - self.init_from_project() + # Will throw an error if there is an issue. + self.init_from_project(config) + + + def update_statistics(self): + """ Recompute frequencies for all name words. + """ + with connect(self.dsn) as conn: + if conn.table_exists('search_name'): + with conn.cursor() as cur: + cur.drop_table("word_frequencies") + LOG.info("Computing word frequencies") + cur.execute("""CREATE TEMP TABLE word_frequencies AS + SELECT unnest(name_vector) as id, count(*) + FROM search_name GROUP BY id""") + cur.execute("CREATE INDEX ON word_frequencies(id)") + LOG.info("Update word table with recomputed frequencies") + cur.execute("""UPDATE word + SET info = info || jsonb_build_object('count', count) + FROM word_frequencies WHERE word_id = id""") + cur.drop_table("word_frequencies") + conn.commit() + + + def _cleanup_housenumbers(self): + """ Remove unused house numbers. + """ + with connect(self.dsn) as conn: + if not conn.table_exists('search_name'): + return + with conn.cursor(name="hnr_counter") as cur: + cur.execute("""SELECT DISTINCT word_id, coalesce(info->>'lookup', word_token) + FROM word + WHERE type = 'H' + AND NOT EXISTS(SELECT * FROM search_name + WHERE ARRAY[word.word_id] && name_vector) + AND (char_length(coalesce(word, word_token)) > 6 + OR coalesce(word, word_token) not similar to '\\d+') + """) + candidates = {token: wid for wid, token in cur} + with conn.cursor(name="hnr_counter") as cur: + cur.execute("""SELECT housenumber FROM placex + WHERE housenumber is not null + AND (char_length(housenumber) > 6 + OR housenumber not similar to '\\d+') + """) + for row in cur: + for hnr in row[0].split(';'): + candidates.pop(hnr, None) + LOG.info("There are %s outdated housenumbers.", len(candidates)) + LOG.debug("Outdated housenumbers: %s", candidates.keys()) + if candidates: + with conn.cursor() as cur: + cur.execute("""DELETE FROM word WHERE word_id = any(%s)""", + (list(candidates.values()), )) + conn.commit() - if self.naming_rules is None: - return "Configuration for tokenizer 'icu' are missing." - return None + + def update_word_tokens(self): + """ Remove unused tokens. + """ + LOG.warning("Cleaning up housenumber tokens.") + self._cleanup_housenumbers() + LOG.warning("Tokenizer house-keeping done.") def name_analyzer(self): @@ -107,19 +171,22 @@ class LegacyICUTokenizer(AbstractTokenizer): Analyzers are not thread-safe. You need to instantiate one per thread. """ - return LegacyICUNameAnalyzer(self.dsn, ICUNameProcessor(self.naming_rules)) + return LegacyICUNameAnalyzer(self.dsn, self.loader.make_sanitizer(), + self.loader.make_token_analysis()) - def _install_php(self, phpdir): + def _install_php(self, phpdir, overwrite=True): """ Install the php script for the tokenizer. """ php_file = self.data_dir / "tokenizer.php" - php_file.write_text(dedent(f"""\ - = 0: - full_names.add(name[:brace_idx].strip()) + if analyzer is None: + postcode_name = item.name.strip().upper() + variant_base = None + else: + postcode_name = analyzer.normalize(item.name) + variant_base = item.get_attr("variant") - return full_names + if variant_base is not None: + postcode = f'{postcode_name}@{variant_base}' + else: + postcode = postcode_name + if postcode not in self._cache.postcodes: + term = self._search_normalized(postcode_name) + if not term: + return - def _add_postcode(self, postcode): - """ Make sure the normalized postcode is present in the word table. - """ - if re.search(r'[:,;]', postcode) is None: - postcode = self.normalize_postcode(postcode) + variants = {term} + if analyzer is not None and variant_base is not None: + variants.update(analyzer.get_variants_ascii(variant_base)) - if postcode not in self._cache.postcodes: - term = self.name_processor.get_search_normalized(postcode) - if not term: - return + with self.conn.cursor() as cur: + cur.execute("SELECT create_postcode_word(%s, %s)", + (postcode, list(variants))) + self._cache.postcodes.add(postcode) - with self.conn.cursor() as cur: - # no word_id needed for postcodes - cur.execute("""INSERT INTO word (word_token, type, word) - (SELECT %s, 'P', pc FROM (VALUES (%s)) as v(pc) - WHERE NOT EXISTS - (SELECT * FROM word - WHERE type = 'P' and word = pc)) - """, (term, postcode)) - self._cache.postcodes.add(postcode) + +class _TokenInfo: + """ Collect token information to be sent back to the database. + """ + def __init__(self): + self.names = None + self.housenumbers = set() + self.housenumber_tokens = set() + self.street_tokens = set() + self.place_tokens = set() + self.address_tokens = {} + self.postcode = None @staticmethod - def _split_housenumbers(hnrs): - if len(hnrs) > 1 or ',' in hnrs[0] or ';' in hnrs[0]: - # split numbers if necessary - simple_list = [] - for hnr in hnrs: - simple_list.extend((x.strip() for x in re.split(r'[;,]', hnr))) - - if len(simple_list) > 1: - hnrs = list(set(simple_list)) - else: - hnrs = simple_list + def _mk_array(tokens): + return f"{{{','.join((str(s) for s in tokens))}}}" - return hnrs + def to_dict(self): + """ Return the token information in database importable format. + """ + out = {} + if self.names: + out['names'] = self.names + if self.housenumbers: + out['hnr'] = ';'.join(self.housenumbers) + out['hnr_tokens'] = self._mk_array(self.housenumber_tokens) -class _TokenInfo: - """ Collect token information to be sent back to the database. - """ - def __init__(self, cache): - self._cache = cache - self.data = {} + if self.street_tokens: + out['street'] = self._mk_array(self.street_tokens) - @staticmethod - def _mk_array(tokens): - return '{%s}' % ','.join((str(s) for s in tokens)) + if self.place_tokens: + out['place'] = self._mk_array(self.place_tokens) + if self.address_tokens: + out['addr'] = self.address_tokens - def add_names(self, fulls, partials): + return out + + + def set_names(self, fulls, partials): """ Adds token information for the normalised names. """ - self.data['names'] = self._mk_array(itertools.chain(fulls, partials)) + self.names = self._mk_array(itertools.chain(fulls, partials)) - def add_housenumbers(self, conn, hnrs): + def add_housenumber(self, token, hnr): """ Extract housenumber information from a list of normalised housenumbers. """ - self.data['hnr_tokens'] = self._mk_array(self._cache.get_hnr_tokens(conn, hnrs)) - self.data['hnr'] = ';'.join(hnrs) + if token: + self.housenumbers.add(hnr) + self.housenumber_tokens.add(token) def add_street(self, tokens): """ Add addr:street match terms. """ - if tokens: - self.data['street'] = self._mk_array(tokens) + self.street_tokens.update(tokens) def add_place(self, tokens): """ Add addr:place search and match terms. """ - if tokens: - self.data['place'] = self._mk_array(tokens) + self.place_tokens.update(tokens) - def add_address_terms(self, terms): + def add_address_term(self, key, partials): """ Add additional address terms. """ - tokens = {key: self._mk_array(partials) - for key, partials in terms if partials} + if partials: + self.address_tokens[key] = self._mk_array(partials) - if tokens: - self.data['addr'] = tokens + def set_postcode(self, postcode): + """ Set the postcode to the given one. + """ + self.postcode = postcode class _TokenCache: @@ -603,31 +726,6 @@ class _TokenCache: def __init__(self): self.names = {} self.partials = {} + self.fulls = {} self.postcodes = set() self.housenumbers = {} - - - def get_hnr_tokens(self, conn, terms): - """ Get token ids for a list of housenumbers, looking them up in the - database if necessary. `terms` is an iterable of normalized - housenumbers. - """ - tokens = [] - askdb = [] - - for term in terms: - token = self.housenumbers.get(term) - if token is None: - askdb.append(term) - else: - tokens.append(token) - - if askdb: - with conn.cursor() as cur: - cur.execute("SELECT nr, getorcreate_hnr_id(nr) FROM unnest(%s) as nr", - (askdb, )) - for term, tid in cur: - self.housenumbers[term] = tid - tokens.append(tid) - - return tokens