X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/fa2bc604685f4d9219844769b733d600d968deff..0da481f207b1aabc5943ea4ab99914d9b1f50b82:/nominatim/tokenizer/legacy_tokenizer.py diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py index b0cbe9c3..0aacb57f 100644 --- a/nominatim/tokenizer/legacy_tokenizer.py +++ b/nominatim/tokenizer/legacy_tokenizer.py @@ -1,7 +1,9 @@ """ Tokenizer implementing normalisation as used before Nominatim 4. """ +from collections import OrderedDict import logging +import re import shutil import psycopg2 @@ -194,6 +196,8 @@ class LegacyNameAnalyzer: self.conn.autocommit = True psycopg2.extras.register_hstore(self.conn) + self._cache = _TokenCache(self.conn) + def __enter__(self): return self @@ -210,10 +214,191 @@ class LegacyNameAnalyzer: self.conn.close() self.conn = None + + def add_postcodes_from_db(self): + """ Add postcodes from the location_postcode table to the word table. + """ + with self.conn.cursor() as cur: + cur.execute("""SELECT count(create_postcode_id(pc)) + FROM (SELECT distinct(postcode) as pc + FROM location_postcode) x""") + def process_place(self, place): """ Determine tokenizer information about the given place. Returns a JSON-serialisable structure that will be handed into the database via the token_info field. """ - return {} + token_info = _TokenInfo(self._cache) + + token_info.add_names(self.conn, place.get('name'), place.get('country_feature')) + + address = place.get('address') + + if address: + self._add_postcode(address.get('postcode')) + token_info.add_housenumbers(self.conn, address) + token_info.add_address_parent(self.conn, address.get('street'), + address.get('place')) + token_info.add_address_parts(self.conn, address) + + return token_info.data + + + def _add_postcode(self, postcode): + """ Make sure the normalized postcode is present in the word table. + """ + if not postcode or re.search(r'[:,;]', postcode) is not None: + return + + def _create_postcode_from_db(pcode): + with self.conn.cursor() as cur: + cur.execute('SELECT create_postcode_id(%s)', (pcode, )) + + self._cache.postcodes.get(postcode.strip().upper(), _create_postcode_from_db) + + +class _TokenInfo: + """ Collect token information to be sent back to the database. + """ + def __init__(self, cache): + self.cache = cache + self.data = {} + + + def add_names(self, conn, names, country_feature): + """ Add token information for the names of the place. + """ + if not names: + return + + with conn.cursor() as cur: + # Create the token IDs for all names. + self.data['names'] = cur.scalar("SELECT make_keywords(%s)::text", + (names, )) + + # Add country tokens to word table if necessary. + if country_feature and re.fullmatch(r'[A-Za-z][A-Za-z]', country_feature): + cur.execute("SELECT create_country(%s, %s)", + (names, country_feature.lower())) + + + def add_housenumbers(self, conn, address): + """ Extract housenumber information from the address. + """ + hnrs = [v for k, v in address.items() + if k in ('housenumber', 'streetnumber', 'conscriptionnumber')] + + if not hnrs: + return + + if len(hnrs) == 1: + token = self.cache.get_housenumber(hnrs[0]) + if token is not None: + self.data['hnr_tokens'] = token + self.data['hnr'] = hnrs[0] + return + + # split numbers if necessary + simple_list = [] + for hnr in hnrs: + simple_list.extend((x.strip() for x in re.split(r'[;,]', hnr))) + + if len(simple_list) > 1: + simple_list = list(set(simple_list)) + + with conn.cursor() as cur: + cur.execute("SELECT (create_housenumbers(%s)).* ", (simple_list, )) + self.data['hnr_tokens'], self.data['hnr'] = cur.fetchone() + + + def add_address_parent(self, conn, street, place): + """ Extract the tokens for street and place terms. + """ + def _get_streetplace(name): + with conn.cursor() as cur: + cur.execute("""SELECT (addr_ids_from_name(%s) || getorcreate_name_id(make_standard_name(%s), ''))::text, + word_ids_from_name(%s)::text""", + (name, name, name)) + return cur.fetchone() + + if street: + self.data['street_search'], self.data['street_match'] = \ + self.cache.streets.get(street, _get_streetplace) + + if place: + self.data['place_search'], self.data['place_match'] = \ + self.cache.streets.get(place, _get_streetplace) + + + def add_address_parts(self, conn, address): + """ Extract address terms. + """ + def _get_address_term(name): + with conn.cursor() as cur: + cur.execute("""SELECT addr_ids_from_name(%s)::text, + word_ids_from_name(%s)::text""", + (name, name)) + return cur.fetchone() + + tokens = {} + for key, value in address.items(): + if not key.startswith('_') and \ + key not in ('country', 'street', 'place', 'postcode', 'full', + 'housenumber', 'streetnumber', 'conscriptionnumber'): + tokens[key] = self.cache.address_terms.get(value, _get_address_term) + + if tokens: + self.data['addr'] = tokens + + +class _LRU: + """ Least recently used cache that accepts a generator function to + produce the item when there is a cache miss. + """ + + def __init__(self, maxsize=128): + self.data = OrderedDict() + self.maxsize = maxsize + + def get(self, key, generator): + """ Get the item with the given key from the cache. If nothing + is found in the cache, generate the value through the + generator function and store it in the cache. + """ + value = self.data.get(key) + if value is not None: + self.data.move_to_end(key) + else: + value = generator(key) + if len(self.data) >= self.maxsize: + self.data.popitem(last=False) + self.data[key] = value + + return value + + +class _TokenCache: + """ Cache for token information to avoid repeated database queries. + + This cache is not thread-safe and needs to be instantiated per + analyzer. + """ + def __init__(self, conn): + # various LRU caches + self.postcodes = _LRU(maxsize=32) + self.streets = _LRU(maxsize=256) + self.places = _LRU(maxsize=128) + self.address_terms = _LRU(maxsize=1024) + + # Lookup houseunumbers up to 100 and cache them + with conn.cursor() as cur: + cur.execute("""SELECT i, ARRAY[getorcreate_housenumber_id(i::text)]::text + FROM generate_series(1, 100) as i""") + self._cached_housenumbers = {str(r[0]) : r[1] for r in cur} + + + def get_housenumber(self, number): + """ Get a housenumber token from the cache. + """ + return self._cached_housenumbers.get(number)