X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/b5540dc35c35c7fa8f01979e972ca429b0b521fb..0da481f207b1aabc5943ea4ab99914d9b1f50b82:/nominatim/tokenizer/legacy_tokenizer.py diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py index 2e05ce54..0aacb57f 100644 --- a/nominatim/tokenizer/legacy_tokenizer.py +++ b/nominatim/tokenizer/legacy_tokenizer.py @@ -1,16 +1,22 @@ """ Tokenizer implementing normalisation as used before Nominatim 4. """ +from collections import OrderedDict import logging +import re import shutil import psycopg2 +import psycopg2.extras from nominatim.db.connection import connect from nominatim.db import properties +from nominatim.db import utils as db_utils +from nominatim.db.sql_preprocessor import SQLPreprocessor from nominatim.errors import UsageError DBCFG_NORMALIZATION = "tokenizer_normalization" +DBCFG_MAXWORDFREQ = "tokenizer_maxwordfreq" LOG = logging.getLogger() @@ -53,6 +59,9 @@ def _install_module(config_module_path, src_dir, module_dir): def _check_module(module_dir, conn): + """ Try to use the PostgreSQL module to confirm that it is correctly + installed and accessible from PostgreSQL. + """ with conn.cursor() as cur: try: cur.execute("""CREATE FUNCTION nominatim_test_import_func(text) @@ -91,7 +100,11 @@ class LegacyTokenizer: with connect(self.dsn) as conn: _check_module(module_dir, conn) - self._save_config(conn) + self._save_config(conn, config) + conn.commit() + + self.update_sql_functions(config) + self._init_db_tables(config) def init_from_project(self): @@ -101,6 +114,19 @@ class LegacyTokenizer: self.normalization = properties.get_property(conn, DBCFG_NORMALIZATION) + def update_sql_functions(self, config): + """ Reimport the SQL functions for this tokenizer. + """ + with connect(self.dsn) as conn: + max_word_freq = properties.get_property(conn, DBCFG_MAXWORDFREQ) + modulepath = config.DATABASE_MODULE_PATH or \ + str((config.project_dir / 'module').resolve()) + sqlp = SQLPreprocessor(conn, config) + sqlp.run_sql_file(conn, 'tokenizer/legacy_tokenizer.sql', + max_word_freq=max_word_freq, + modulepath=modulepath) + + def migrate_database(self, config): """ Initialise the project directory of an existing database for use with this tokenizer. @@ -114,11 +140,265 @@ class LegacyTokenizer: with connect(self.dsn) as conn: _check_module(module_dir, conn) - self._save_config(conn) + self._save_config(conn, config) + + + def name_analyzer(self): + """ Create a new analyzer for tokenizing names and queries + using this tokinzer. Analyzers are context managers and should + be used accordingly: + + ``` + with tokenizer.name_analyzer() as analyzer: + analyser.tokenize() + ``` + + When used outside the with construct, the caller must ensure to + call the close() function before destructing the analyzer. + + Analyzers are not thread-safe. You need to instantiate one per thread. + """ + return LegacyNameAnalyzer(self.dsn) + + + def _init_db_tables(self, config): + """ Set up the word table and fill it with pre-computed word + frequencies. + """ + with connect(self.dsn) as conn: + sqlp = SQLPreprocessor(conn, config) + sqlp.run_sql_file(conn, 'tokenizer/legacy_tokenizer_tables.sql') + conn.commit() + + LOG.warning("Precomputing word tokens") + db_utils.execute_file(self.dsn, config.lib_dir.data / 'words.sql') - def _save_config(self, conn): + def _save_config(self, conn, config): """ Save the configuration that needs to remain stable for the given database as database properties. """ properties.set_property(conn, DBCFG_NORMALIZATION, self.normalization) + properties.set_property(conn, DBCFG_MAXWORDFREQ, config.MAX_WORD_FREQUENCY) + + + +class LegacyNameAnalyzer: + """ The legacy analyzer uses the special Postgresql module for + splitting names. + + Each instance opens a connection to the database to request the + normalization. + """ + + def __init__(self, dsn): + self.conn = connect(dsn).connection + self.conn.autocommit = True + psycopg2.extras.register_hstore(self.conn) + + self._cache = _TokenCache(self.conn) + + + def __enter__(self): + return self + + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + + def close(self): + """ Free all resources used by the analyzer. + """ + if self.conn: + self.conn.close() + self.conn = None + + + def add_postcodes_from_db(self): + """ Add postcodes from the location_postcode table to the word table. + """ + with self.conn.cursor() as cur: + cur.execute("""SELECT count(create_postcode_id(pc)) + FROM (SELECT distinct(postcode) as pc + FROM location_postcode) x""") + + def process_place(self, place): + """ Determine tokenizer information about the given place. + + Returns a JSON-serialisable structure that will be handed into + the database via the token_info field. + """ + token_info = _TokenInfo(self._cache) + + token_info.add_names(self.conn, place.get('name'), place.get('country_feature')) + + address = place.get('address') + + if address: + self._add_postcode(address.get('postcode')) + token_info.add_housenumbers(self.conn, address) + token_info.add_address_parent(self.conn, address.get('street'), + address.get('place')) + token_info.add_address_parts(self.conn, address) + + return token_info.data + + + def _add_postcode(self, postcode): + """ Make sure the normalized postcode is present in the word table. + """ + if not postcode or re.search(r'[:,;]', postcode) is not None: + return + + def _create_postcode_from_db(pcode): + with self.conn.cursor() as cur: + cur.execute('SELECT create_postcode_id(%s)', (pcode, )) + + self._cache.postcodes.get(postcode.strip().upper(), _create_postcode_from_db) + + +class _TokenInfo: + """ Collect token information to be sent back to the database. + """ + def __init__(self, cache): + self.cache = cache + self.data = {} + + + def add_names(self, conn, names, country_feature): + """ Add token information for the names of the place. + """ + if not names: + return + + with conn.cursor() as cur: + # Create the token IDs for all names. + self.data['names'] = cur.scalar("SELECT make_keywords(%s)::text", + (names, )) + + # Add country tokens to word table if necessary. + if country_feature and re.fullmatch(r'[A-Za-z][A-Za-z]', country_feature): + cur.execute("SELECT create_country(%s, %s)", + (names, country_feature.lower())) + + + def add_housenumbers(self, conn, address): + """ Extract housenumber information from the address. + """ + hnrs = [v for k, v in address.items() + if k in ('housenumber', 'streetnumber', 'conscriptionnumber')] + + if not hnrs: + return + + if len(hnrs) == 1: + token = self.cache.get_housenumber(hnrs[0]) + if token is not None: + self.data['hnr_tokens'] = token + self.data['hnr'] = hnrs[0] + return + + # split numbers if necessary + simple_list = [] + for hnr in hnrs: + simple_list.extend((x.strip() for x in re.split(r'[;,]', hnr))) + + if len(simple_list) > 1: + simple_list = list(set(simple_list)) + + with conn.cursor() as cur: + cur.execute("SELECT (create_housenumbers(%s)).* ", (simple_list, )) + self.data['hnr_tokens'], self.data['hnr'] = cur.fetchone() + + + def add_address_parent(self, conn, street, place): + """ Extract the tokens for street and place terms. + """ + def _get_streetplace(name): + with conn.cursor() as cur: + cur.execute("""SELECT (addr_ids_from_name(%s) || getorcreate_name_id(make_standard_name(%s), ''))::text, + word_ids_from_name(%s)::text""", + (name, name, name)) + return cur.fetchone() + + if street: + self.data['street_search'], self.data['street_match'] = \ + self.cache.streets.get(street, _get_streetplace) + + if place: + self.data['place_search'], self.data['place_match'] = \ + self.cache.streets.get(place, _get_streetplace) + + + def add_address_parts(self, conn, address): + """ Extract address terms. + """ + def _get_address_term(name): + with conn.cursor() as cur: + cur.execute("""SELECT addr_ids_from_name(%s)::text, + word_ids_from_name(%s)::text""", + (name, name)) + return cur.fetchone() + + tokens = {} + for key, value in address.items(): + if not key.startswith('_') and \ + key not in ('country', 'street', 'place', 'postcode', 'full', + 'housenumber', 'streetnumber', 'conscriptionnumber'): + tokens[key] = self.cache.address_terms.get(value, _get_address_term) + + if tokens: + self.data['addr'] = tokens + + +class _LRU: + """ Least recently used cache that accepts a generator function to + produce the item when there is a cache miss. + """ + + def __init__(self, maxsize=128): + self.data = OrderedDict() + self.maxsize = maxsize + + def get(self, key, generator): + """ Get the item with the given key from the cache. If nothing + is found in the cache, generate the value through the + generator function and store it in the cache. + """ + value = self.data.get(key) + if value is not None: + self.data.move_to_end(key) + else: + value = generator(key) + if len(self.data) >= self.maxsize: + self.data.popitem(last=False) + self.data[key] = value + + return value + + +class _TokenCache: + """ Cache for token information to avoid repeated database queries. + + This cache is not thread-safe and needs to be instantiated per + analyzer. + """ + def __init__(self, conn): + # various LRU caches + self.postcodes = _LRU(maxsize=32) + self.streets = _LRU(maxsize=256) + self.places = _LRU(maxsize=128) + self.address_terms = _LRU(maxsize=1024) + + # Lookup houseunumbers up to 100 and cache them + with conn.cursor() as cur: + cur.execute("""SELECT i, ARRAY[getorcreate_housenumber_id(i::text)]::text + FROM generate_series(1, 100) as i""") + self._cached_housenumbers = {str(r[0]) : r[1] for r in cur} + + + def get_housenumber(self, number): + """ Get a housenumber token from the cache. + """ + return self._cached_housenumbers.get(number)