"""
Tokenizer implementing normalisation as used before Nominatim 4.
"""
+from collections import OrderedDict
import logging
+import re
import shutil
import psycopg2
+import psycopg2.extras
from nominatim.db.connection import connect
from nominatim.db import properties
+from nominatim.db import utils as db_utils
+from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.errors import UsageError
DBCFG_NORMALIZATION = "tokenizer_normalization"
+DBCFG_MAXWORDFREQ = "tokenizer_maxwordfreq"
LOG = logging.getLogger()
def _check_module(module_dir, conn):
+ """ Try to use the PostgreSQL module to confirm that it is correctly
+ installed and accessible from PostgreSQL.
+ """
with conn.cursor() as cur:
try:
cur.execute("""CREATE FUNCTION nominatim_test_import_func(text)
with connect(self.dsn) as conn:
_check_module(module_dir, conn)
- self._save_config(conn)
+ self._save_config(conn, config)
+ conn.commit()
+
+ self.update_sql_functions(config)
+ self._init_db_tables(config)
def init_from_project(self):
self.normalization = properties.get_property(conn, DBCFG_NORMALIZATION)
+ def update_sql_functions(self, config):
+ """ Reimport the SQL functions for this tokenizer.
+ """
+ with connect(self.dsn) as conn:
+ max_word_freq = properties.get_property(conn, DBCFG_MAXWORDFREQ)
+ modulepath = config.DATABASE_MODULE_PATH or \
+ str((config.project_dir / 'module').resolve())
+ sqlp = SQLPreprocessor(conn, config)
+ sqlp.run_sql_file(conn, 'tokenizer/legacy_tokenizer.sql',
+ max_word_freq=max_word_freq,
+ modulepath=modulepath)
+
+
def migrate_database(self, config):
""" Initialise the project directory of an existing database for
use with this tokenizer.
with connect(self.dsn) as conn:
_check_module(module_dir, conn)
- self._save_config(conn)
+ self._save_config(conn, config)
+
+
+ def name_analyzer(self):
+ """ Create a new analyzer for tokenizing names and queries
+ using this tokinzer. Analyzers are context managers and should
+ be used accordingly:
+
+ ```
+ with tokenizer.name_analyzer() as analyzer:
+ analyser.tokenize()
+ ```
+
+ When used outside the with construct, the caller must ensure to
+ call the close() function before destructing the analyzer.
+
+ Analyzers are not thread-safe. You need to instantiate one per thread.
+ """
+ return LegacyNameAnalyzer(self.dsn)
+
+
+ def _init_db_tables(self, config):
+ """ Set up the word table and fill it with pre-computed word
+ frequencies.
+ """
+ with connect(self.dsn) as conn:
+ sqlp = SQLPreprocessor(conn, config)
+ sqlp.run_sql_file(conn, 'tokenizer/legacy_tokenizer_tables.sql')
+ conn.commit()
+
+ LOG.warning("Precomputing word tokens")
+ db_utils.execute_file(self.dsn, config.lib_dir.data / 'words.sql')
- def _save_config(self, conn):
+ def _save_config(self, conn, config):
""" Save the configuration that needs to remain stable for the given
database as database properties.
"""
properties.set_property(conn, DBCFG_NORMALIZATION, self.normalization)
+ properties.set_property(conn, DBCFG_MAXWORDFREQ, config.MAX_WORD_FREQUENCY)
+
+
+
+class LegacyNameAnalyzer:
+ """ The legacy analyzer uses the special Postgresql module for
+ splitting names.
+
+ Each instance opens a connection to the database to request the
+ normalization.
+ """
+
+ def __init__(self, dsn):
+ self.conn = connect(dsn).connection
+ self.conn.autocommit = True
+ psycopg2.extras.register_hstore(self.conn)
+
+ self._cache = _TokenCache(self.conn)
+
+
+ def __enter__(self):
+ return self
+
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+
+ def close(self):
+ """ Free all resources used by the analyzer.
+ """
+ if self.conn:
+ self.conn.close()
+ self.conn = None
+
+
+ def add_postcodes_from_db(self):
+ """ Add postcodes from the location_postcode table to the word table.
+ """
+ with self.conn.cursor() as cur:
+ cur.execute("""SELECT count(create_postcode_id(pc))
+ FROM (SELECT distinct(postcode) as pc
+ FROM location_postcode) x""")
+
+ def process_place(self, place):
+ """ Determine tokenizer information about the given place.
+
+ Returns a JSON-serialisable structure that will be handed into
+ the database via the token_info field.
+ """
+ token_info = _TokenInfo(self._cache)
+
+ token_info.add_names(self.conn, place.get('name'), place.get('country_feature'))
+
+ address = place.get('address')
+
+ if address:
+ self._add_postcode(address.get('postcode'))
+ token_info.add_housenumbers(self.conn, address)
+ token_info.add_address_parent(self.conn, address.get('street'),
+ address.get('place'))
+ token_info.add_address_parts(self.conn, address)
+
+ return token_info.data
+
+
+ def _add_postcode(self, postcode):
+ """ Make sure the normalized postcode is present in the word table.
+ """
+ if not postcode or re.search(r'[:,;]', postcode) is not None:
+ return
+
+ def _create_postcode_from_db(pcode):
+ with self.conn.cursor() as cur:
+ cur.execute('SELECT create_postcode_id(%s)', (pcode, ))
+
+ self._cache.postcodes.get(postcode.strip().upper(), _create_postcode_from_db)
+
+
+class _TokenInfo:
+ """ Collect token information to be sent back to the database.
+ """
+ def __init__(self, cache):
+ self.cache = cache
+ self.data = {}
+
+
+ def add_names(self, conn, names, country_feature):
+ """ Add token information for the names of the place.
+ """
+ if not names:
+ return
+
+ with conn.cursor() as cur:
+ # Create the token IDs for all names.
+ self.data['names'] = cur.scalar("SELECT make_keywords(%s)::text",
+ (names, ))
+
+ # Add country tokens to word table if necessary.
+ if country_feature and re.fullmatch(r'[A-Za-z][A-Za-z]', country_feature):
+ cur.execute("SELECT create_country(%s, %s)",
+ (names, country_feature.lower()))
+
+
+ def add_housenumbers(self, conn, address):
+ """ Extract housenumber information from the address.
+ """
+ hnrs = [v for k, v in address.items()
+ if k in ('housenumber', 'streetnumber', 'conscriptionnumber')]
+
+ if not hnrs:
+ return
+
+ if len(hnrs) == 1:
+ token = self.cache.get_housenumber(hnrs[0])
+ if token is not None:
+ self.data['hnr_tokens'] = token
+ self.data['hnr'] = hnrs[0]
+ return
+
+ # split numbers if necessary
+ simple_list = []
+ for hnr in hnrs:
+ simple_list.extend((x.strip() for x in re.split(r'[;,]', hnr)))
+
+ if len(simple_list) > 1:
+ simple_list = list(set(simple_list))
+
+ with conn.cursor() as cur:
+ cur.execute("SELECT (create_housenumbers(%s)).* ", (simple_list, ))
+ self.data['hnr_tokens'], self.data['hnr'] = cur.fetchone()
+
+
+ def add_address_parent(self, conn, street, place):
+ """ Extract the tokens for street and place terms.
+ """
+ def _get_streetplace(name):
+ with conn.cursor() as cur:
+ cur.execute("""SELECT (addr_ids_from_name(%s) || getorcreate_name_id(make_standard_name(%s), ''))::text,
+ word_ids_from_name(%s)::text""",
+ (name, name, name))
+ return cur.fetchone()
+
+ if street:
+ self.data['street_search'], self.data['street_match'] = \
+ self.cache.streets.get(street, _get_streetplace)
+
+ if place:
+ self.data['place_search'], self.data['place_match'] = \
+ self.cache.streets.get(place, _get_streetplace)
+
+
+ def add_address_parts(self, conn, address):
+ """ Extract address terms.
+ """
+ def _get_address_term(name):
+ with conn.cursor() as cur:
+ cur.execute("""SELECT addr_ids_from_name(%s)::text,
+ word_ids_from_name(%s)::text""",
+ (name, name))
+ return cur.fetchone()
+
+ tokens = {}
+ for key, value in address.items():
+ if not key.startswith('_') and \
+ key not in ('country', 'street', 'place', 'postcode', 'full',
+ 'housenumber', 'streetnumber', 'conscriptionnumber'):
+ tokens[key] = self.cache.address_terms.get(value, _get_address_term)
+
+ if tokens:
+ self.data['addr'] = tokens
+
+
+class _LRU:
+ """ Least recently used cache that accepts a generator function to
+ produce the item when there is a cache miss.
+ """
+
+ def __init__(self, maxsize=128):
+ self.data = OrderedDict()
+ self.maxsize = maxsize
+
+ def get(self, key, generator):
+ """ Get the item with the given key from the cache. If nothing
+ is found in the cache, generate the value through the
+ generator function and store it in the cache.
+ """
+ value = self.data.get(key)
+ if value is not None:
+ self.data.move_to_end(key)
+ else:
+ value = generator(key)
+ if len(self.data) >= self.maxsize:
+ self.data.popitem(last=False)
+ self.data[key] = value
+
+ return value
+
+
+class _TokenCache:
+ """ Cache for token information to avoid repeated database queries.
+
+ This cache is not thread-safe and needs to be instantiated per
+ analyzer.
+ """
+ def __init__(self, conn):
+ # various LRU caches
+ self.postcodes = _LRU(maxsize=32)
+ self.streets = _LRU(maxsize=256)
+ self.places = _LRU(maxsize=128)
+ self.address_terms = _LRU(maxsize=1024)
+
+ # Lookup houseunumbers up to 100 and cache them
+ with conn.cursor() as cur:
+ cur.execute("""SELECT i, ARRAY[getorcreate_housenumber_id(i::text)]::text
+ FROM generate_series(1, 100) as i""")
+ self._cached_housenumbers = {str(r[0]) : r[1] for r in cur}
+
+
+ def get_housenumber(self, number):
+ """ Get a housenumber token from the cache.
+ """
+ return self._cached_housenumbers.get(number)
projects / nominatim.git / blobdiff