X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/2c909c1f0c7c6e6a176eb5cc15e8d6a45449ebe8..a9edd57fe28021e3526e6787a43eda4dd3b84125:/nominatim/indexer/indexer.py?ds=sidebyside diff --git a/nominatim/indexer/indexer.py b/nominatim/indexer/indexer.py index 6e0ed60f..233423f0 100644 --- a/nominatim/indexer/indexer.py +++ b/nominatim/indexer/indexer.py @@ -1,199 +1,242 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. """ Main work horse for indexing (computing addresses) the database. """ -# pylint: disable=C0111 +from typing import Optional, Any, cast import logging -import select +import time -import psycopg2 +import psycopg2.extras -from .progress import ProgressLogger -from ..db.async_connection import DBConnection +from nominatim.tokenizer.base import AbstractTokenizer +from nominatim.indexer.progress import ProgressLogger +from nominatim.indexer import runners +from nominatim.db.async_connection import DBConnection, WorkerPool +from nominatim.db.connection import connect, Connection, Cursor +from nominatim.typing import DictCursorResults LOG = logging.getLogger() -class RankRunner: - """ Returns SQL commands for indexing one rank within the placex table. + +class PlaceFetcher: + """ Asynchronous connection that fetches place details for processing. """ + def __init__(self, dsn: str, setup_conn: Connection) -> None: + self.wait_time = 0.0 + self.current_ids: Optional[DictCursorResults] = None + self.conn: Optional[DBConnection] = DBConnection(dsn, + cursor_factory=psycopg2.extras.DictCursor) + + with setup_conn.cursor() as cur: + # need to fetch those manually because register_hstore cannot + # fetch them on an asynchronous connection below. + hstore_oid = cur.scalar("SELECT 'hstore'::regtype::oid") + hstore_array_oid = cur.scalar("SELECT 'hstore[]'::regtype::oid") + + psycopg2.extras.register_hstore(self.conn.conn, oid=hstore_oid, + array_oid=hstore_array_oid) + + def close(self) -> None: + """ Close the underlying asynchronous connection. + """ + if self.conn: + self.conn.close() + self.conn = None - def __init__(self, rank): - self.rank = rank - def name(self): - return "rank {}".format(self.rank) + def fetch_next_batch(self, cur: Cursor, runner: runners.Runner) -> bool: + """ Send a request for the next batch of places. + If details for the places are required, they will be fetched + asynchronously. - def sql_count_objects(self): - return """SELECT count(*) FROM placex - WHERE rank_address = {} and indexed_status > 0 - """.format(self.rank) + Returns true if there is still data available. + """ + ids = cast(Optional[DictCursorResults], cur.fetchmany(100)) - def sql_get_objects(self): - return """SELECT place_id FROM placex - WHERE indexed_status > 0 and rank_address = {} - ORDER BY geometry_sector""".format(self.rank) + if not ids: + self.current_ids = None + return False - @staticmethod - def sql_index_place(ids): - return "UPDATE placex SET indexed_status = 0 WHERE place_id IN ({})"\ - .format(','.join((str(i) for i in ids))) + assert self.conn is not None + self.current_ids = runner.get_place_details(self.conn, ids) + return True -class InterpolationRunner: - """ Returns SQL commands for indexing the address interpolation table - location_property_osmline. - """ + def get_batch(self) -> DictCursorResults: + """ Get the next batch of data, previously requested with + `fetch_next_batch`. + """ + assert self.conn is not None + assert self.conn.cursor is not None - @staticmethod - def name(): - return "interpolation lines (location_property_osmline)" - - @staticmethod - def sql_count_objects(): - return """SELECT count(*) FROM location_property_osmline - WHERE indexed_status > 0""" - - @staticmethod - def sql_get_objects(): - return """SELECT place_id FROM location_property_osmline - WHERE indexed_status > 0 - ORDER BY geometry_sector""" - - @staticmethod - def sql_index_place(ids): - return """UPDATE location_property_osmline - SET indexed_status = 0 WHERE place_id IN ({})"""\ - .format(','.join((str(i) for i in ids))) - -class BoundaryRunner: - """ Returns SQL commands for indexing the administrative boundaries - of a certain rank. - """ + if self.current_ids is not None and not self.current_ids: + tstart = time.time() + self.conn.wait() + self.wait_time += time.time() - tstart + self.current_ids = cast(Optional[DictCursorResults], + self.conn.cursor.fetchall()) - def __init__(self, rank): - self.rank = rank + return self.current_ids if self.current_ids is not None else [] - def name(self): - return "boundaries rank {}".format(self.rank) + def __enter__(self) -> 'PlaceFetcher': + return self - def sql_count_objects(self): - return """SELECT count(*) FROM placex - WHERE indexed_status > 0 - AND rank_search = {} - AND class = 'boundary' and type = 'administrative'""".format(self.rank) - def sql_get_objects(self): - return """SELECT place_id FROM placex - WHERE indexed_status > 0 and rank_search = {} - and class = 'boundary' and type = 'administrative' - ORDER BY partition, admin_level""".format(self.rank) + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + assert self.conn is not None + self.conn.wait() + self.close() - @staticmethod - def sql_index_place(ids): - return "UPDATE placex SET indexed_status = 0 WHERE place_id IN ({})"\ - .format(','.join((str(i) for i in ids))) class Indexer: """ Main indexing routine. """ - def __init__(self, dsn, num_threads): - self.conn = psycopg2.connect(dsn) - self.threads = [DBConnection(dsn) for _ in range(num_threads)] + def __init__(self, dsn: str, tokenizer: AbstractTokenizer, num_threads: int): + self.dsn = dsn + self.tokenizer = tokenizer + self.num_threads = num_threads + + + def has_pending(self) -> bool: + """ Check if any data still needs indexing. + This function must only be used after the import has finished. + Otherwise it will be very expensive. + """ + with connect(self.dsn) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 'a' FROM placex WHERE indexed_status > 0 LIMIT 1") + return cur.rowcount > 0 + + + def index_full(self, analyse: bool = True) -> None: + """ Index the complete database. This will first index boundaries + followed by all other objects. When `analyse` is True, then the + database will be analysed at the appropriate places to + ensure that database statistics are updated. + """ + with connect(self.dsn) as conn: + conn.autocommit = True + + def _analyze() -> None: + if analyse: + with conn.cursor() as cur: + cur.execute('ANALYZE') + + if self.index_by_rank(0, 4) > 0: + _analyze() + + if self.index_boundaries(0, 30) > 100: + _analyze() + + if self.index_by_rank(5, 25) > 100: + _analyze() - def index_boundaries(self, minrank, maxrank): + if self.index_by_rank(26, 30) > 1000: + _analyze() + + if self.index_postcodes() > 100: + _analyze() + + + def index_boundaries(self, minrank: int, maxrank: int) -> int: + """ Index only administrative boundaries within the given rank range. + """ + total = 0 LOG.warning("Starting indexing boundaries using %s threads", - len(self.threads)) + self.num_threads) - for rank in range(max(minrank, 4), min(maxrank, 26)): - self.index(BoundaryRunner(rank)) + with self.tokenizer.name_analyzer() as analyzer: + for rank in range(max(minrank, 4), min(maxrank, 26)): + total += self._index(runners.BoundaryRunner(rank, analyzer)) - def index_by_rank(self, minrank, maxrank): - """ Run classic indexing by rank. + return total + + def index_by_rank(self, minrank: int, maxrank: int) -> int: + """ Index all entries of placex in the given rank range (inclusive) + in order of their address rank. + + When rank 30 is requested then also interpolations and + places with address rank 0 will be indexed. """ + total = 0 maxrank = min(maxrank, 30) LOG.warning("Starting indexing rank (%i to %i) using %i threads", - minrank, maxrank, len(self.threads)) + minrank, maxrank, self.num_threads) + + with self.tokenizer.name_analyzer() as analyzer: + for rank in range(max(1, minrank), maxrank + 1): + total += self._index(runners.RankRunner(rank, analyzer), 20 if rank == 30 else 1) + + if maxrank == 30: + total += self._index(runners.RankRunner(0, analyzer)) + total += self._index(runners.InterpolationRunner(analyzer), 20) - for rank in range(max(1, minrank), maxrank): - self.index(RankRunner(rank)) + return total - if maxrank == 30: - self.index(RankRunner(0)) - self.index(InterpolationRunner(), 20) - self.index(RankRunner(30), 20) - else: - self.index(RankRunner(maxrank)) - def update_status_table(self): + def index_postcodes(self) -> int: + """Index the entries of the location_postcode table. + """ + LOG.warning("Starting indexing postcodes using %s threads", self.num_threads) + + return self._index(runners.PostcodeRunner(), 20) + + + def update_status_table(self) -> None: """ Update the status in the status table to 'indexed'. """ - with self.conn.cursor() as cur: - cur.execute('UPDATE import_status SET indexed = true') - self.conn.commit() + with connect(self.dsn) as conn: + with conn.cursor() as cur: + cur.execute('UPDATE import_status SET indexed = true') - def index(self, obj, batch=1): - """ Index a single rank or table. `obj` describes the SQL to use + conn.commit() + + def _index(self, runner: runners.Runner, batch: int = 1) -> int: + """ Index a single rank or table. `runner` describes the SQL to use for indexing. `batch` describes the number of objects that should be processed with a single SQL statement """ - LOG.warning("Starting %s (using batch size %s)", obj.name(), batch) - - cur = self.conn.cursor() - cur.execute(obj.sql_count_objects()) + LOG.warning("Starting %s (using batch size %s)", runner.name(), batch) - total_tuples = cur.fetchone()[0] - LOG.debug("Total number of rows: %i", total_tuples) + with connect(self.dsn) as conn: + psycopg2.extras.register_hstore(conn) + with conn.cursor() as cur: + total_tuples = cur.scalar(runner.sql_count_objects()) + LOG.debug("Total number of rows: %i", total_tuples) - cur.close() + conn.commit() - progress = ProgressLogger(obj.name(), total_tuples) + progress = ProgressLogger(runner.name(), total_tuples) - if total_tuples > 0: - cur = self.conn.cursor(name='places') - cur.execute(obj.sql_get_objects()) + if total_tuples > 0: + with conn.cursor(name='places') as cur: + cur.execute(runner.sql_get_objects()) - next_thread = self.find_free_thread() - while True: - places = [p[0] for p in cur.fetchmany(batch)] - if not places: - break + with PlaceFetcher(self.dsn, conn) as fetcher: + with WorkerPool(self.dsn, self.num_threads) as pool: + has_more = fetcher.fetch_next_batch(cur, runner) + while has_more: + places = fetcher.get_batch() - LOG.debug("Processing places: %s", str(places)) - thread = next(next_thread) + # asynchronously get the next batch + has_more = fetcher.fetch_next_batch(cur, runner) - thread.perform(obj.sql_index_place(places)) - progress.add(len(places)) + # And insert the current batch + for idx in range(0, len(places), batch): + part = places[idx:idx + batch] + LOG.debug("Processing places: %s", str(part)) + runner.index_places(pool.next_free_worker(), part) + progress.add(len(part)) - cur.close() + LOG.info("Wait time: fetcher: %.2fs, pool: %.2fs", + fetcher.wait_time, pool.wait_time) - for thread in self.threads: - thread.wait() + conn.commit() - progress.done() - - def find_free_thread(self): - """ Generator that returns the next connection that is free for - sending a query. - """ - ready = self.threads - command_stat = 0 - - while True: - for thread in ready: - if thread.is_done(): - command_stat += 1 - yield thread - - # refresh the connections occasionaly to avoid potential - # memory leaks in Postgresql. - if command_stat > 100000: - for thread in self.threads: - while not thread.is_done(): - thread.wait() - thread.connect() - command_stat = 0 - ready = self.threads - else: - ready, _, _ = select.select(self.threads, [], []) - - assert False, "Unreachable code" + return progress.done()