X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/9036bf3398fac5778f3d3242b2da58a3b16a3a0d..5360384b7024fb2a1f84fee55511beef41c3fd46:/nominatim/api/core.py diff --git a/nominatim/api/core.py b/nominatim/api/core.py index 3e4b175a..0f1dd715 100644 --- a/nominatim/api/core.py +++ b/nominatim/api/core.py @@ -7,15 +7,16 @@ """ Implementation of classes for API access via libraries. """ -from typing import Mapping, Optional, Any, AsyncIterator, Dict, Sequence +from typing import Mapping, Optional, Any, AsyncIterator, Dict, Sequence, List, Tuple import asyncio +import sys import contextlib from pathlib import Path import sqlalchemy as sa import sqlalchemy.ext.asyncio as sa_asyncio - +from nominatim.errors import UsageError from nominatim.db.sqlalchemy_schema import SearchTables from nominatim.db.async_core_library import PGCORE_LIB, PGCORE_ERROR from nominatim.config import Configuration @@ -23,7 +24,8 @@ from nominatim.api.connection import SearchConnection from nominatim.api.status import get_status, StatusResult from nominatim.api.lookup import get_detailed_place, get_simple_place from nominatim.api.reverse import ReverseGeocoder -from nominatim.api.types import PlaceRef, LookupDetails, AnyPoint, DataLayer +from nominatim.api.search import ForwardGeocoder, Phrase, PhraseType, make_query_analyzer +import nominatim.api.types as ntyp from nominatim.api.results import DetailedResult, ReverseResult, SearchResults @@ -31,11 +33,17 @@ class NominatimAPIAsync: """ API loader asynchornous version. """ def __init__(self, project_dir: Path, - environ: Optional[Mapping[str, str]] = None) -> None: + environ: Optional[Mapping[str, str]] = None, + loop: Optional[asyncio.AbstractEventLoop] = None) -> None: self.config = Configuration(project_dir, environ) + self.query_timeout = self.config.get_int('QUERY_TIMEOUT') \ + if self.config.QUERY_TIMEOUT else None self.server_version = 0 - self._engine_lock = asyncio.Lock() + if sys.version_info >= (3, 10): + self._engine_lock = asyncio.Lock() + else: + self._engine_lock = asyncio.Lock(loop=loop) # pylint: disable=unexpected-keyword-arg self._engine: Optional[sa_asyncio.AsyncEngine] = None self._tables: Optional[SearchTables] = None self._property_cache: Dict[str, Any] = {'DB:server_version': 0} @@ -53,11 +61,10 @@ class NominatimAPIAsync: return dsn = self.config.get_database_params() + pool_size = self.config.get_int('API_POOL_SIZE') query = {k: v for k, v in dsn.items() if k not in ('user', 'password', 'dbname', 'host', 'port')} - if PGCORE_LIB == 'asyncpg': - query['prepared_statement_cache_size'] = '0' dburl = sa.engine.URL.create( f'postgresql+{PGCORE_LIB}', @@ -65,7 +72,9 @@ class NominatimAPIAsync: username=dsn.get('user'), password=dsn.get('password'), host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None, query=query) - engine = sa_asyncio.create_async_engine(dburl, future=True) + engine = sa_asyncio.create_async_engine(dburl, future=True, + max_overflow=0, pool_size=pool_size, + echo=self.config.get_bool('DEBUG_SQL')) try: async with engine.begin() as conn: @@ -121,6 +130,7 @@ class NominatimAPIAsync: """ try: async with self.begin() as conn: + conn.set_query_timeout(self.query_timeout) status = await get_status(conn) except (PGCORE_ERROR, sa.exc.OperationalError): return StatusResult(700, 'Database connection failed') @@ -128,32 +138,34 @@ class NominatimAPIAsync: return status - async def details(self, place: PlaceRef, - details: Optional[LookupDetails] = None) -> Optional[DetailedResult]: + async def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]: """ Get detailed information about a place in the database. Returns None if there is no entry under the given ID. """ + details = ntyp.LookupDetails.from_kwargs(params) async with self.begin() as conn: - return await get_detailed_place(conn, place, details or LookupDetails()) + conn.set_query_timeout(self.query_timeout) + if details.keywords: + await make_query_analyzer(conn) + return await get_detailed_place(conn, place, details) - async def lookup(self, places: Sequence[PlaceRef], - details: Optional[LookupDetails] = None) -> SearchResults: + async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults: """ Get simple information about a list of places. Returns a list of place information for all IDs that were found. """ - if details is None: - details = LookupDetails() + details = ntyp.LookupDetails.from_kwargs(params) async with self.begin() as conn: + conn.set_query_timeout(self.query_timeout) + if details.keywords: + await make_query_analyzer(conn) return SearchResults(filter(None, [await get_simple_place(conn, p, details) for p in places])) - async def reverse(self, coord: AnyPoint, max_rank: Optional[int] = None, - layer: Optional[DataLayer] = None, - details: Optional[LookupDetails] = None) -> Optional[ReverseResult]: + async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]: """ Find a place by its coordinates. Also known as reverse geocoding. Returns the closest result that can be found or None if @@ -164,15 +176,115 @@ class NominatimAPIAsync: # There are no results to be expected outside valid coordinates. return None - if layer is None: - layer = DataLayer.ADDRESS | DataLayer.POI + details = ntyp.ReverseDetails.from_kwargs(params) + async with self.begin() as conn: + conn.set_query_timeout(self.query_timeout) + if details.keywords: + await make_query_analyzer(conn) + geocoder = ReverseGeocoder(conn, details) + return await geocoder.lookup(coord) + + + async def search(self, query: str, **params: Any) -> SearchResults: + """ Find a place by free-text search. Also known as forward geocoding. + """ + query = query.strip() + if not query: + raise UsageError('Nothing to search for.') - max_rank = max(0, min(max_rank or 30, 30)) + async with self.begin() as conn: + conn.set_query_timeout(self.query_timeout) + geocoder = ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params), + self.config.get_int('REQUEST_TIMEOUT') \ + if self.config.REQUEST_TIMEOUT else None) + phrases = [Phrase(PhraseType.NONE, p.strip()) for p in query.split(',')] + return await geocoder.lookup(phrases) + + + # pylint: disable=too-many-arguments,too-many-branches + async def search_address(self, amenity: Optional[str] = None, + street: Optional[str] = None, + city: Optional[str] = None, + county: Optional[str] = None, + state: Optional[str] = None, + country: Optional[str] = None, + postalcode: Optional[str] = None, + **params: Any) -> SearchResults: + """ Find an address using structured search. + """ + async with self.begin() as conn: + conn.set_query_timeout(self.query_timeout) + details = ntyp.SearchDetails.from_kwargs(params) + + phrases: List[Phrase] = [] + + if amenity: + phrases.append(Phrase(PhraseType.AMENITY, amenity)) + if street: + phrases.append(Phrase(PhraseType.STREET, street)) + if city: + phrases.append(Phrase(PhraseType.CITY, city)) + if county: + phrases.append(Phrase(PhraseType.COUNTY, county)) + if state: + phrases.append(Phrase(PhraseType.STATE, state)) + if postalcode: + phrases.append(Phrase(PhraseType.POSTCODE, postalcode)) + if country: + phrases.append(Phrase(PhraseType.COUNTRY, country)) + + if not phrases: + raise UsageError('Nothing to search for.') + + if amenity or street: + details.restrict_min_max_rank(26, 30) + elif city: + details.restrict_min_max_rank(13, 25) + elif county: + details.restrict_min_max_rank(10, 12) + elif state: + details.restrict_min_max_rank(5, 9) + elif postalcode: + details.restrict_min_max_rank(5, 11) + else: + details.restrict_min_max_rank(4, 4) + + if 'layers' not in params: + details.layers = ntyp.DataLayer.ADDRESS + if amenity: + details.layers |= ntyp.DataLayer.POI + + geocoder = ForwardGeocoder(conn, details, + self.config.get_int('REQUEST_TIMEOUT') \ + if self.config.REQUEST_TIMEOUT else None) + return await geocoder.lookup(phrases) + + + async def search_category(self, categories: List[Tuple[str, str]], + near_query: Optional[str] = None, + **params: Any) -> SearchResults: + """ Find an object of a certain category near another place. + The near place may either be given as an unstructured search + query in itself or as coordinates. + """ + if not categories: + return SearchResults() + details = ntyp.SearchDetails.from_kwargs(params) async with self.begin() as conn: - geocoder = ReverseGeocoder(conn, max_rank, layer, - details or LookupDetails()) - return await geocoder.lookup(coord) + conn.set_query_timeout(self.query_timeout) + if near_query: + phrases = [Phrase(PhraseType.NONE, p) for p in near_query.split(',')] + else: + phrases = [] + if details.keywords: + await make_query_analyzer(conn) + + geocoder = ForwardGeocoder(conn, details, + self.config.get_int('REQUEST_TIMEOUT') \ + if self.config.REQUEST_TIMEOUT else None) + return await geocoder.lookup_pois(categories, phrases) + class NominatimAPI: @@ -182,7 +294,7 @@ class NominatimAPI: def __init__(self, project_dir: Path, environ: Optional[Mapping[str, str]] = None) -> None: self._loop = asyncio.new_event_loop() - self._async_api = NominatimAPIAsync(project_dir, environ) + self._async_api = NominatimAPIAsync(project_dir, environ, loop=self._loop) def close(self) -> None: @@ -206,29 +318,59 @@ class NominatimAPI: return self._loop.run_until_complete(self._async_api.status()) - def details(self, place: PlaceRef, - details: Optional[LookupDetails] = None) -> Optional[DetailedResult]: + def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]: """ Get detailed information about a place in the database. """ - return self._loop.run_until_complete(self._async_api.details(place, details)) + return self._loop.run_until_complete(self._async_api.details(place, **params)) - def lookup(self, places: Sequence[PlaceRef], - details: Optional[LookupDetails] = None) -> SearchResults: + def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults: """ Get simple information about a list of places. Returns a list of place information for all IDs that were found. """ - return self._loop.run_until_complete(self._async_api.lookup(places, details)) + return self._loop.run_until_complete(self._async_api.lookup(places, **params)) - def reverse(self, coord: AnyPoint, max_rank: Optional[int] = None, - layer: Optional[DataLayer] = None, - details: Optional[LookupDetails] = None) -> Optional[ReverseResult]: + def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]: """ Find a place by its coordinates. Also known as reverse geocoding. Returns the closest result that can be found or None if no place matches the given criteria. """ + return self._loop.run_until_complete(self._async_api.reverse(coord, **params)) + + + def search(self, query: str, **params: Any) -> SearchResults: + """ Find a place by free-text search. Also known as forward geocoding. + """ + return self._loop.run_until_complete( + self._async_api.search(query, **params)) + + + # pylint: disable=too-many-arguments + def search_address(self, amenity: Optional[str] = None, + street: Optional[str] = None, + city: Optional[str] = None, + county: Optional[str] = None, + state: Optional[str] = None, + country: Optional[str] = None, + postalcode: Optional[str] = None, + **params: Any) -> SearchResults: + """ Find an address using structured search. + """ + return self._loop.run_until_complete( + self._async_api.search_address(amenity, street, city, county, + state, country, postalcode, **params)) + + + def search_category(self, categories: List[Tuple[str, str]], + near_query: Optional[str] = None, + **params: Any) -> SearchResults: + """ Find an object of a certain category near another place. + The near place may either be given as an unstructured search + query in itself or as a geographic area through the + viewbox or near parameters. + """ return self._loop.run_until_complete( - self._async_api.reverse(coord, max_rank, layer, details)) + self._async_api.search_category(categories, near_query, **params))