update instructions for Vagrant

[nominatim.git] / src / nominatim_api / core.py

diff --git a/src/nominatim_api/core.py b/src/nominatim_api/core.py
index 6f91e7a8e4a1574f8cc44d2975a9f57e609d2cf3..3cf9e989141b5f118fe8d287805ad754124d42df 100644 (file)
--- a/src/nominatim_api/core.py
+++ b/src/nominatim_api/core.py
@@ -7,7 +7,8 @@
 """
 Implementation of classes for API access via libraries.
 """
 """
 Implementation of classes for API access via libraries.
 """

-from typing import Mapping, Optional, Any, AsyncIterator, Dict, Sequence, List, Tuple
+from typing import Mapping, Optional, Any, AsyncIterator, Dict, Sequence, List, \
+                   Union, Tuple, cast

 import asyncio
 import sys
 import contextlib
 import asyncio
 import sys
 import contextlib


@@ -16,21 +17,21 @@ from pathlib import Path
 import sqlalchemy as sa
 import sqlalchemy.ext.asyncio as sa_asyncio
 
 import sqlalchemy as sa
 import sqlalchemy.ext.asyncio as sa_asyncio
 

-from nominatim_core.errors import UsageError
-from nominatim_core.db.sqlalchemy_schema import SearchTables
-from nominatim_core.db.async_core_library import PGCORE_LIB, PGCORE_ERROR
-from nominatim_core.config import Configuration
-from .sql import sqlite_functions, sqlalchemy_functions #pylint: disable=unused-import
+from .errors import UsageError
+from .sql.sqlalchemy_schema import SearchTables
+from .sql.async_core_library import PGCORE_LIB, PGCORE_ERROR
+from .config import Configuration
+from .sql import sqlite_functions, sqlalchemy_functions  # noqa

 from .connection import SearchConnection
 from .status import get_status, StatusResult
 from .connection import SearchConnection
 from .status import get_status, StatusResult

-from .lookup import get_detailed_place, get_simple_place
+from .lookup import get_places, get_detailed_place

 from .reverse import ReverseGeocoder
 from .search import ForwardGeocoder, Phrase, PhraseType, make_query_analyzer
 from . import types as ntyp
 from .results import DetailedResult, ReverseResult, SearchResults
 
 
 from .reverse import ReverseGeocoder
 from .search import ForwardGeocoder, Phrase, PhraseType, make_query_analyzer
 from . import types as ntyp
 from .results import DetailedResult, ReverseResult, SearchResults
 
 

-class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
+class NominatimAPIAsync:

     """ The main frontend to the Nominatim database implements the
         functions for lookup, forward and reverse geocoding using
         asynchronous functions.
     """ The main frontend to the Nominatim database implements the
         functions for lookup, forward and reverse geocoding using
         asynchronous functions.


@@ -38,8 +39,10 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
         This class shares most of the functions with its synchronous
         version. There are some additional functions or parameters,
         which are documented below.
         This class shares most of the functions with its synchronous
         version. There are some additional functions or parameters,
         which are documented below.

+
+        This class should usually be used as a context manager in 'with' context.

     """
     """

-    def __init__(self, project_dir: Path,
+    def __init__(self, project_dir: Optional[Union[str, Path]] = None,

                  environ: Optional[Mapping[str, str]] = None,
                  loop: Optional[asyncio.AbstractEventLoop] = None) -> None:
         """ Initiate a new frontend object with synchronous API functions.
                  environ: Optional[Mapping[str, str]] = None,
                  loop: Optional[asyncio.AbstractEventLoop] = None) -> None:
         """ Initiate a new frontend object with synchronous API functions.


@@ -58,19 +61,18 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
         """
         self.config = Configuration(project_dir, environ)
         self.query_timeout = self.config.get_int('QUERY_TIMEOUT') \
         """
         self.config = Configuration(project_dir, environ)
         self.query_timeout = self.config.get_int('QUERY_TIMEOUT') \

-                             if self.config.QUERY_TIMEOUT else None
+            if self.config.QUERY_TIMEOUT else None

         self.reverse_restrict_to_country_area = self.config.get_bool('SEARCH_WITHIN_COUNTRIES')
         self.server_version = 0
 
         if sys.version_info >= (3, 10):
             self._engine_lock = asyncio.Lock()
         else:
         self.reverse_restrict_to_country_area = self.config.get_bool('SEARCH_WITHIN_COUNTRIES')
         self.server_version = 0
 
         if sys.version_info >= (3, 10):
             self._engine_lock = asyncio.Lock()
         else:

-            self._engine_lock = asyncio.Lock(loop=loop) # pylint: disable=unexpected-keyword-arg
+            self._engine_lock = asyncio.Lock(loop=loop)

         self._engine: Optional[sa_asyncio.AsyncEngine] = None
         self._tables: Optional[SearchTables] = None
         self._property_cache: Dict[str, Any] = {'DB:server_version': 0}
 
         self._engine: Optional[sa_asyncio.AsyncEngine] = None
         self._tables: Optional[SearchTables] = None
         self._property_cache: Dict[str, Any] = {'DB:server_version': 0}
 

-

     async def setup_database(self) -> None:
         """ Set up the SQL engine and connections.
 
     async def setup_database(self) -> None:
         """ Set up the SQL engine and connections.
 


@@ -92,7 +94,6 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
                 extra_args['max_overflow'] = 0
                 extra_args['pool_size'] = self.config.get_int('API_POOL_SIZE')
 
                 extra_args['max_overflow'] = 0
                 extra_args['pool_size'] = self.config.get_int('API_POOL_SIZE')
 

-

             is_sqlite = self.config.DATABASE_DSN.startswith('sqlite:')
 
             if is_sqlite:
             is_sqlite = self.config.DATABASE_DSN.startswith('sqlite:')
 
             if is_sqlite:


@@ -107,16 +108,16 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
                     raise UsageError(f"SQlite database '{params.get('dbname')}' does not exist.")
             else:
                 dsn = self.config.get_database_params()
                     raise UsageError(f"SQlite database '{params.get('dbname')}' does not exist.")
             else:
                 dsn = self.config.get_database_params()

-                query = {k: v for k, v in dsn.items()
+                query = {k: str(v) for k, v in dsn.items()

                          if k not in ('user', 'password', 'dbname', 'host', 'port')}
 
                 dburl = sa.engine.URL.create(
                            f'postgresql+{PGCORE_LIB}',
                          if k not in ('user', 'password', 'dbname', 'host', 'port')}
 
                 dburl = sa.engine.URL.create(
                            f'postgresql+{PGCORE_LIB}',

-                           database=dsn.get('dbname'),
-                           username=dsn.get('user'),
-                           password=dsn.get('password'),
-                           host=dsn.get('host'),
-                           port=int(dsn['port']) if 'port' in dsn else None,
+                           database=cast(str, dsn.get('dbname')),
+                           username=cast(str, dsn.get('user')),
+                           password=cast(str, dsn.get('password')),
+                           host=cast(str, dsn.get('host')),
+                           port=int(cast(str, dsn['port'])) if 'port' in dsn else None,

                            query=query)
 
             engine = sa_asyncio.create_async_engine(dburl, **extra_args)
                            query=query)
 
             engine = sa_asyncio.create_async_engine(dburl, **extra_args)


@@ -137,26 +138,23 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
                     async with engine.begin() as conn:
                         result = await conn.scalar(sa.text('SHOW server_version_num'))
                         server_version = int(result)
                     async with engine.begin() as conn:
                         result = await conn.scalar(sa.text('SHOW server_version_num'))
                         server_version = int(result)

-                        if server_version >= 110000:
-                            await conn.execute(sa.text("SET jit_above_cost TO '-1'"))
-                            await conn.execute(sa.text(
-                                    "SET max_parallel_workers_per_gather TO '0'"))
+                        await conn.execute(sa.text("SET jit_above_cost TO '-1'"))
+                        await conn.execute(sa.text(
+                                "SET max_parallel_workers_per_gather TO '0'"))

                 except (PGCORE_ERROR, sa.exc.OperationalError):
                     server_version = 0
 
                 except (PGCORE_ERROR, sa.exc.OperationalError):
                     server_version = 0
 

-                if server_version >= 110000:
-                    @sa.event.listens_for(engine.sync_engine, "connect")
-                    def _on_connect(dbapi_con: Any, _: Any) -> None:
-                        cursor = dbapi_con.cursor()
-                        cursor.execute("SET jit_above_cost TO '-1'")
-                        cursor.execute("SET max_parallel_workers_per_gather TO '0'")
+                @sa.event.listens_for(engine.sync_engine, "connect")
+                def _on_connect(dbapi_con: Any, _: Any) -> None:
+                    cursor = dbapi_con.cursor()
+                    cursor.execute("SET jit_above_cost TO '-1'")
+                    cursor.execute("SET max_parallel_workers_per_gather TO '0'")

 
             self._property_cache['DB:server_version'] = server_version
 
 
             self._property_cache['DB:server_version'] = server_version
 

-            self._tables = SearchTables(sa.MetaData()) # pylint: disable=no-member
+            self._tables = SearchTables(sa.MetaData())

             self._engine = engine
 
             self._engine = engine
 

-

     async def close(self) -> None:
         """ Close all active connections to the database. The NominatimAPIAsync
             object remains usable after closing. If a new API functions is
     async def close(self) -> None:
         """ Close all active connections to the database. The NominatimAPIAsync
             object remains usable after closing. If a new API functions is


@@ -165,6 +163,11 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
         if self._engine is not None:
             await self._engine.dispose()
 
         if self._engine is not None:
             await self._engine.dispose()
 

+    async def __aenter__(self) -> 'NominatimAPIAsync':
+        return self
+
+    async def __aexit__(self, *_: Any) -> None:
+        await self.close()

 
     @contextlib.asynccontextmanager
     async def begin(self) -> AsyncIterator[SearchConnection]:
 
     @contextlib.asynccontextmanager
     async def begin(self) -> AsyncIterator[SearchConnection]:


@@ -183,7 +186,6 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
         async with self._engine.begin() as conn:
             yield SearchConnection(conn, self._tables, self._property_cache)
 
         async with self._engine.begin() as conn:
             yield SearchConnection(conn, self._tables, self._property_cache)
 

-

     async def status(self) -> StatusResult:
         """ Return the status of the database.
         """
     async def status(self) -> StatusResult:
         """ Return the status of the database.
         """


@@ -196,7 +198,6 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
 
         return status
 
 
         return status
 

-

     async def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]:
         """ Get detailed information about a place in the database.
 
     async def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]:
         """ Get detailed information about a place in the database.
 


@@ -209,7 +210,6 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
                 await make_query_analyzer(conn)
             return await get_detailed_place(conn, place, details)
 
                 await make_query_analyzer(conn)
             return await get_detailed_place(conn, place, details)
 

-

     async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
         """ Get simple information about a list of places.
 
     async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
         """ Get simple information about a list of places.
 


@@ -220,9 +220,7 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
                 await make_query_analyzer(conn)
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
                 await make_query_analyzer(conn)

-            return SearchResults(filter(None,
-                                        [await get_simple_place(conn, p, details) for p in places]))
-
+            return await get_places(conn, places, details)

 
     async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
         """ Find a place by its coordinates. Also known as reverse geocoding.
 
     async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
         """ Find a place by its coordinates. Also known as reverse geocoding.


@@ -244,7 +242,6 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
                                        self.reverse_restrict_to_country_area)
             return await geocoder.lookup(coord)
 
                                        self.reverse_restrict_to_country_area)
             return await geocoder.lookup(coord)
 

-

     async def search(self, query: str, **params: Any) -> SearchResults:
         """ Find a place by free-text search. Also known as forward geocoding.
         """
     async def search(self, query: str, **params: Any) -> SearchResults:
         """ Find a place by free-text search. Also known as forward geocoding.
         """


@@ -255,13 +252,11 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             geocoder = ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             geocoder = ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),

-                                       self.config.get_int('REQUEST_TIMEOUT') \
-                                         if self.config.REQUEST_TIMEOUT else None)
+                                       self.config.get_int('REQUEST_TIMEOUT')
+                                       if self.config.REQUEST_TIMEOUT else None)

             phrases = [Phrase(PhraseType.NONE, p.strip()) for p in query.split(',')]
             return await geocoder.lookup(phrases)
 
             phrases = [Phrase(PhraseType.NONE, p.strip()) for p in query.split(',')]
             return await geocoder.lookup(phrases)
 

-
-    # pylint: disable=too-many-arguments,too-many-branches

     async def search_address(self, amenity: Optional[str] = None,
                              street: Optional[str] = None,
                              city: Optional[str] = None,
     async def search_address(self, amenity: Optional[str] = None,
                              street: Optional[str] = None,
                              city: Optional[str] = None,


@@ -315,11 +310,10 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
                     details.layers |= ntyp.DataLayer.POI
 
             geocoder = ForwardGeocoder(conn, details,
                     details.layers |= ntyp.DataLayer.POI
 
             geocoder = ForwardGeocoder(conn, details,

-                                       self.config.get_int('REQUEST_TIMEOUT') \
-                                         if self.config.REQUEST_TIMEOUT else None)
+                                       self.config.get_int('REQUEST_TIMEOUT')
+                                       if self.config.REQUEST_TIMEOUT else None)

             return await geocoder.lookup(phrases)
 
             return await geocoder.lookup(phrases)
 

-

     async def search_category(self, categories: List[Tuple[str, str]],
                               near_query: Optional[str] = None,
                               **params: Any) -> SearchResults:
     async def search_category(self, categories: List[Tuple[str, str]],
                               near_query: Optional[str] = None,
                               **params: Any) -> SearchResults:


@@ -341,19 +335,20 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
                     await make_query_analyzer(conn)
 
             geocoder = ForwardGeocoder(conn, details,
                     await make_query_analyzer(conn)
 
             geocoder = ForwardGeocoder(conn, details,

-                                       self.config.get_int('REQUEST_TIMEOUT') \
-                                         if self.config.REQUEST_TIMEOUT else None)
+                                       self.config.get_int('REQUEST_TIMEOUT')
+                                       if self.config.REQUEST_TIMEOUT else None)

             return await geocoder.lookup_pois(categories, phrases)
 
 
             return await geocoder.lookup_pois(categories, phrases)
 
 

-

 class NominatimAPI:
     """ This class provides a thin synchronous wrapper around the asynchronous
         Nominatim functions. It creates its own event loop and runs each
         synchronous function call to completion using that loop.
 class NominatimAPI:
     """ This class provides a thin synchronous wrapper around the asynchronous
         Nominatim functions. It creates its own event loop and runs each
         synchronous function call to completion using that loop.

+
+        This class should usually be used as a context manager in 'with' context.

     """
 
     """
 

-    def __init__(self, project_dir: Path,
+    def __init__(self, project_dir: Optional[Union[str, Path]] = None,

                  environ: Optional[Mapping[str, str]] = None) -> None:
         """ Initiate a new frontend object with synchronous API functions.
 
                  environ: Optional[Mapping[str, str]] = None) -> None:
         """ Initiate a new frontend object with synchronous API functions.
 


@@ -369,20 +364,25 @@ class NominatimAPI:
         self._loop = asyncio.new_event_loop()
         self._async_api = NominatimAPIAsync(project_dir, environ, loop=self._loop)
 
         self._loop = asyncio.new_event_loop()
         self._async_api = NominatimAPIAsync(project_dir, environ, loop=self._loop)
 

-

     def close(self) -> None:
         """ Close all active connections to the database.
 
             This function also closes the asynchronous worker loop making
             the NominatimAPI object unusable.
         """
     def close(self) -> None:
         """ Close all active connections to the database.
 
             This function also closes the asynchronous worker loop making
             the NominatimAPI object unusable.
         """

-        self._loop.run_until_complete(self._async_api.close())
-        self._loop.close()
+        if not self._loop.is_closed():
+            self._loop.run_until_complete(self._async_api.close())
+            self._loop.close()
+
+    def __enter__(self) -> 'NominatimAPI':
+        return self

 
 

+    def __exit__(self, *_: Any) -> None:
+        self.close()

 
     @property
     def config(self) -> Configuration:
 
     @property
     def config(self) -> Configuration:

-        """ Provide read-only access to the [configuration](#Configuration)
+        """ Provide read-only access to the [configuration](Configuration.md)

             used by the API.
         """
         return self._async_api.config
             used by the API.
         """
         return self._async_api.config


@@ -405,7 +405,6 @@ class NominatimAPI:
         """
         return self._loop.run_until_complete(self._async_api.status())
 
         """
         return self._loop.run_until_complete(self._async_api.status())
 

-

     def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]:
         """ Get detailed information about a place in the database.
 
     def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]:
         """ Get detailed information about a place in the database.
 


@@ -488,7 +487,6 @@ class NominatimAPI:
         """
         return self._loop.run_until_complete(self._async_api.details(place, **params))
 
         """
         return self._loop.run_until_complete(self._async_api.details(place, **params))
 

-

     def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
         """ Get simple information about a list of places.
 
     def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
         """ Get simple information about a list of places.
 


@@ -565,7 +563,6 @@ class NominatimAPI:
         """
         return self._loop.run_until_complete(self._async_api.lookup(places, **params))
 
         """
         return self._loop.run_until_complete(self._async_api.lookup(places, **params))
 

-

     def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
         """ Find a place by its coordinates. Also known as reverse geocoding.
 
     def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
         """ Find a place by its coordinates. Also known as reverse geocoding.
 


@@ -647,7 +644,6 @@ class NominatimAPI:
         """
         return self._loop.run_until_complete(self._async_api.reverse(coord, **params))
 
         """
         return self._loop.run_until_complete(self._async_api.reverse(coord, **params))
 

-

     def search(self, query: str, **params: Any) -> SearchResults:
         """ Find a place by free-text search. Also known as forward geocoding.
 
     def search(self, query: str, **params: Any) -> SearchResults:
         """ Find a place by free-text search. Also known as forward geocoding.
 


@@ -747,8 +743,6 @@ class NominatimAPI:
         return self._loop.run_until_complete(
                    self._async_api.search(query, **params))
 
         return self._loop.run_until_complete(
                    self._async_api.search(query, **params))
 

-
-    # pylint: disable=too-many-arguments

     def search_address(self, amenity: Optional[str] = None,
                        street: Optional[str] = None,
                        city: Optional[str] = None,
     def search_address(self, amenity: Optional[str] = None,
                        street: Optional[str] = None,
                        city: Optional[str] = None,


@@ -866,7 +860,6 @@ class NominatimAPI:
                    self._async_api.search_address(amenity, street, city, county,
                                                   state, country, postalcode, **params))
 
                    self._async_api.search_address(amenity, street, city, county,
                                                   state, country, postalcode, **params))
 

-

     def search_category(self, categories: List[Tuple[str, str]],
                         near_query: Optional[str] = None,
                         **params: Any) -> SearchResults:
     def search_category(self, categories: List[Tuple[str, str]],
                         near_query: Optional[str] = None,
                         **params: Any) -> SearchResults: