]> git.openstreetmap.org Git - nominatim.git/commitdiff
Merge remote-tracking branch 'upstream/master'
authorSarah Hoffmann <lonvia@denofr.de>
Fri, 17 Feb 2023 14:52:31 +0000 (15:52 +0100)
committerSarah Hoffmann <lonvia@denofr.de>
Fri, 17 Feb 2023 14:52:31 +0000 (15:52 +0100)
16 files changed:
.github/workflows/ci-tests.yml
docs/develop/Development-Environment.md
lib-php/ReverseGeocode.php
lib-sql/indices.sql
lib-sql/tables.sql
nominatim/api/connection.py
nominatim/api/logging.py [new file with mode: 0644]
nominatim/api/lookup.py
nominatim/api/results.py
nominatim/api/types.py
nominatim/api/v1/format.py
nominatim/api/v1/server_glue.py
nominatim/tools/migration.py
nominatim/version.py
test/python/api/conftest.py
test/python/cli/test_cmd_api.py

index 3fe0b9d81d326925b64024f3567e0e0676e5c241..7dfb3f1d34b0b677940abf7695986fcd87070001 100644 (file)
@@ -74,6 +74,8 @@ jobs:
                   php-version: ${{ matrix.php }}
                   tools: phpunit:9, phpcs, composer
                   ini-values: opcache.jit=disable
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
             - uses: actions/setup-python@v4
               with:
@@ -138,7 +140,7 @@ jobs:
               working-directory: Nominatim/test/bdd
 
             - name: Install mypy and typechecking info
-              run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson typing-extensions
+              run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson types-Pygments typing-extensions
               if: matrix.flavour != 'oldstuff'
 
             - name: Python static typechecking
index 7284e99ef1390593208db2a91066e77c454317e2..3234b8cbf3124574591b662606a406d7b75d27e3 100644 (file)
@@ -65,7 +65,7 @@ sudo apt install php-cgi phpunit php-codesniffer \
 
 pip3 install --user behave mkdocs mkdocstrings pytest pytest-asyncio pylint \
                     mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \
-                    types-ujson types-requests typing-extensions\
+                    types-ujson types-requests types-Pygments typing-extensions\
                     sanic-testing httpx asgi-lifespan
 ```
 
index d12e4da3c8d65c4ae9ba02f10a4a0d333d1e77f6..712f14809bb07f0edffc3b0e371cf791119380db 100644 (file)
@@ -122,12 +122,13 @@ class ReverseGeocode
                 $sSQL .= ' FROM placex';
                 $sSQL .= ' WHERE osm_type = \'N\'';
                 $sSQL .= ' AND country_code = \''.$sCountryCode.'\'';
-                $sSQL .= ' AND rank_search < 26 '; // needed to select right index
+                $sSQL .= ' AND rank_address between 4 and 25'; // needed to select right index
                 $sSQL .= ' AND rank_search between 5 and ' .min(25, $iMaxRank);
-                $sSQL .= ' AND class = \'place\' AND type != \'postcode\'';
+                $sSQL .= ' AND type != \'postcode\'';
                 $sSQL .= ' AND name IS NOT NULL ';
                 $sSQL .= ' and indexed_status = 0 and linked_place_id is null';
-                $sSQL .= ' AND ST_DWithin('.$sPointSQL.', geometry, 1.8)) p ';
+                $sSQL .= ' AND ST_Buffer(geometry, reverse_place_diameter(rank_search)) && '.$sPointSQL;
+                $sSQL .= ') as a ';
                 $sSQL .= 'WHERE distance <= reverse_place_diameter(rank_search)';
                 $sSQL .= ' ORDER BY rank_search DESC, distance ASC';
                 $sSQL .= ' LIMIT 1';
@@ -216,23 +217,18 @@ class ReverseGeocode
                 $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';
                 $sSQL .= ' FROM placex';
                 $sSQL .= ' WHERE osm_type = \'N\'';
-                // using rank_search because of a better differentiation
-                // for place nodes at rank_address 16
                 $sSQL .= ' AND rank_search > '.$iRankSearch;
                 $sSQL .= ' AND rank_search <= '.$iMaxRank;
-                $sSQL .= ' AND rank_search < 26 '; // needed to select right index
-                $sSQL .= ' AND rank_address > 0';
-                $sSQL .= ' AND class = \'place\'';
+                $sSQL .= ' AND rank_address between 4 and 25';  // needed to select right index
                 $sSQL .= ' AND type != \'postcode\'';
                 $sSQL .= ' AND name IS NOT NULL ';
                 $sSQL .= ' AND indexed_status = 0 AND linked_place_id is null';
-                $sSQL .= ' AND ST_DWithin('.$sPointSQL.', geometry, reverse_place_diameter('.$iRankSearch.'::smallint))';
-                $sSQL .= ' ORDER BY distance ASC,';
-                $sSQL .= ' rank_address DESC';
-                $sSQL .= ' limit 500) as a';
-                $sSQL .= ' WHERE ST_CONTAINS((SELECT geometry FROM placex WHERE place_id = '.$iPlaceID.'), geometry )';
+                $sSQL .= ' AND ST_Buffer(geometry, reverse_place_diameter(rank_search)) && '.$sPointSQL;
+                $sSQL .= ' ORDER BY rank_search DESC, distance ASC';
+                $sSQL .= ' limit 100) as a';
+                $sSQL .= ' WHERE ST_Contains((SELECT geometry FROM placex WHERE place_id = '.$iPlaceID.'), geometry )';
                 $sSQL .= ' AND distance <= reverse_place_diameter(rank_search)';
-                $sSQL .= ' ORDER BY distance ASC, rank_search DESC';
+                $sSQL .= ' ORDER BY rank_search DESC, distance ASC';
                 $sSQL .= ' LIMIT 1';
                 Debug::printSQL($sSQL);
 
index 9130fb52998f4188dce449b96f2bd132b76abdee..ed078895ee8901473ac0f613b97e6d9cabe8c88e 100644 (file)
@@ -30,6 +30,13 @@ CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPolygon
     AND rank_address between 4 and 25 AND type != 'postcode'
     AND name is not null AND indexed_status = 0 AND linked_place_id is null;
 ---
+-- used in reverse large area lookup
+CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPlaceNode
+  ON placex USING gist (ST_Buffer(geometry, reverse_place_diameter(rank_search)))
+  {{db.tablespace.search_index}}
+  WHERE rank_address between 4 and 25 AND type != 'postcode'
+    AND name is not null AND linked_place_id is null AND osm_type = 'N';
+---
 CREATE INDEX IF NOT EXISTS idx_osmline_parent_place_id
   ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}}
   WHERE parent_place_id is not null;
index d576485e33839ea4b7ccbe163bab1bf530b4c9bc..17216b50990dfd2b24e3b47b8969dac5904109c2 100644 (file)
@@ -190,7 +190,6 @@ CREATE INDEX idx_placex_geometry_buildings ON placex
 
 -- Usage: - linking of similar named places to boundaries
 --        - linking of place nodes with same type to boundaries
---        - lookupPolygon()
 CREATE INDEX idx_placex_geometry_placenode ON placex
   USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}}
   WHERE osm_type = 'N' and rank_search < 26
index 79a5e3470d7b43340500aaa64c8739986e3467d4..efa4490e0cbf6d263398a4cf21d2f883b419bad9 100644 (file)
@@ -13,6 +13,7 @@ import sqlalchemy as sa
 from sqlalchemy.ext.asyncio import AsyncConnection
 
 from nominatim.db.sqlalchemy_schema import SearchTables
+from nominatim.api.logging import log
 
 class SearchConnection:
     """ An extended SQLAlchemy connection class, that also contains
@@ -34,14 +35,16 @@ class SearchConnection:
                     ) -> Any:
         """ Execute a 'scalar()' query on the connection.
         """
+        log().sql(self.connection, sql)
         return await self.connection.scalar(sql, params)
 
 
-    async def execute(self, sql: sa.sql.base.Executable,
+    async def execute(self, sql: 'sa.Executable',
                       params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None] = None
-                     ) -> 'sa.engine.Result[Any]':
+                     ) -> 'sa.Result[Any]':
         """ Execute a 'execute()' query on the connection.
         """
+        log().sql(self.connection, sql)
         return await self.connection.execute(sql, params)
 
 
diff --git a/nominatim/api/logging.py b/nominatim/api/logging.py
new file mode 100644 (file)
index 0000000..e9c8847
--- /dev/null
@@ -0,0 +1,240 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Functions for specialised logging with HTML output.
+"""
+from typing import Any, cast
+from contextvars import ContextVar
+import textwrap
+import io
+
+import sqlalchemy as sa
+from sqlalchemy.ext.asyncio import AsyncConnection
+
+try:
+    from pygments import highlight
+    from pygments.lexers import PythonLexer, PostgresLexer
+    from pygments.formatters import HtmlFormatter
+    CODE_HIGHLIGHT = True
+except ModuleNotFoundError:
+    CODE_HIGHLIGHT = False
+
+
+class BaseLogger:
+    """ Interface for logging function.
+
+        The base implementation does nothing. Overwrite the functions
+        in derived classes which implement logging functionality.
+    """
+    def get_buffer(self) -> str:
+        """ Return the current content of the log buffer.
+        """
+        return ''
+
+    def function(self, func: str, **kwargs: Any) -> None:
+        """ Start a new debug chapter for the given function and its parameters.
+        """
+
+
+    def section(self, heading: str) -> None:
+        """ Start a new section with the given title.
+        """
+
+
+    def comment(self, text: str) -> None:
+        """ Add a simple comment to the debug output.
+        """
+
+
+    def var_dump(self, heading: str, var: Any) -> None:
+        """ Print the content of the variable to the debug output prefixed by
+            the given heading.
+        """
+
+
+    def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None:
+        """ Print the SQL for the given statement.
+        """
+
+
+class HTMLLogger(BaseLogger):
+    """ Logger that formats messages in HTML.
+    """
+    def __init__(self) -> None:
+        self.buffer = io.StringIO()
+
+
+    def get_buffer(self) -> str:
+        return HTML_HEADER + self.buffer.getvalue() + HTML_FOOTER
+
+
+    def function(self, func: str, **kwargs: Any) -> None:
+        self._write(f"<h1>Debug output for {func}()</h1>\n<p>Parameters:<dl>")
+        for name, value in kwargs.items():
+            self._write(f'<dt>{name}</dt><dd>{self._python_var(value)}</dd>')
+        self._write('</dl></p>')
+
+
+    def section(self, heading: str) -> None:
+        self._write(f"<h2>{heading}</h2>")
+
+
+    def comment(self, text: str) -> None:
+        self._write(f"<p>{text}</p>")
+
+
+    def var_dump(self, heading: str, var: Any) -> None:
+        self._write(f'<h5>{heading}</h5>{self._python_var(var)}')
+
+
+    def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None:
+        sqlstr = str(cast('sa.ClauseElement', statement)
+                      .compile(conn.sync_engine, compile_kwargs={"literal_binds": True}))
+        if CODE_HIGHLIGHT:
+            sqlstr = highlight(sqlstr, PostgresLexer(),
+                               HtmlFormatter(nowrap=True, lineseparator='<br>'))
+            self._write(f'<div class="highlight"><code class="lang-sql">{sqlstr}</code></div>')
+        else:
+            self._write(f'<code class="lang-sql">{sqlstr}</code>')
+
+
+    def _python_var(self, var: Any) -> str:
+        if CODE_HIGHLIGHT:
+            fmt = highlight(repr(var), PythonLexer(), HtmlFormatter(nowrap=True))
+            return f'<div class="highlight"><code class="lang-python">{fmt}</code></div>'
+
+        return f'<code class="lang-python">{str(var)}</code>'
+
+
+    def _write(self, text: str) -> None:
+        """ Add the raw text to the debug output.
+        """
+        self.buffer.write(text)
+
+
+class TextLogger(BaseLogger):
+    """ Logger creating output suitable for the console.
+    """
+    def __init__(self) -> None:
+        self.buffer = io.StringIO()
+
+
+    def get_buffer(self) -> str:
+        return self.buffer.getvalue()
+
+
+    def function(self, func: str, **kwargs: Any) -> None:
+        self._write(f"#### Debug output for {func}()\n\nParameters:\n")
+        for name, value in kwargs.items():
+            self._write(f'  {name}: {self._python_var(value)}\n')
+        self._write('\n')
+
+
+    def section(self, heading: str) -> None:
+        self._write(f"\n# {heading}\n\n")
+
+
+    def comment(self, text: str) -> None:
+        self._write(f"{text}\n")
+
+
+    def var_dump(self, heading: str, var: Any) -> None:
+        self._write(f'{heading}:\n  {self._python_var(var)}\n\n')
+
+
+    def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None:
+        sqlstr = str(cast('sa.ClauseElement', statement)
+                      .compile(conn.sync_engine, compile_kwargs={"literal_binds": True}))
+        sqlstr = '\n| '.join(textwrap.wrap(sqlstr, width=78))
+        self._write(f"| {sqlstr}\n\n")
+
+
+    def _python_var(self, var: Any) -> str:
+        return str(var)
+
+
+    def _write(self, text: str) -> None:
+        self.buffer.write(text)
+
+
+logger: ContextVar[BaseLogger] = ContextVar('logger', default=BaseLogger())
+
+
+def set_log_output(fmt: str) -> None:
+    """ Enable collecting debug information.
+    """
+    if fmt == 'html':
+        logger.set(HTMLLogger())
+    elif fmt == 'text':
+        logger.set(TextLogger())
+    else:
+        logger.set(BaseLogger())
+
+
+def log() -> BaseLogger:
+    """ Return the logger for the current context.
+    """
+    return logger.get()
+
+
+def get_and_disable() -> str:
+    """ Return the current content of the debug buffer and disable logging.
+    """
+    buf = logger.get().get_buffer()
+    logger.set(BaseLogger())
+    return buf
+
+
+HTML_HEADER: str = """<!DOCTYPE html>
+<html>
+<head>
+  <title>Nominatim - Debug</title>
+  <style>
+""" + \
+(HtmlFormatter(nobackground=True).get_style_defs('.highlight') if CODE_HIGHLIGHT else '') +\
+"""
+    h2 { font-size: x-large }
+
+    dl {
+      padding-left: 10pt;
+      font-family: monospace
+    }
+
+    dt {
+      float: left;
+      font-weight: bold;
+      margin-right: 0.5em
+    }
+
+    dt::after { content: ": "; }
+
+    dd::after {
+      clear: left;
+      display: block
+    }
+
+    .lang-sql {
+      color: #555;
+      font-size: small
+    }
+
+    h5 {
+        border: solid lightgrey 0.1pt;
+        margin-bottom: 0;
+        background-color: #f7f7f7
+    }
+
+    h5 + .highlight {
+        padding: 3pt;
+        border: solid lightgrey 0.1pt
+    }
+  </style>
+</head>
+<body>
+"""
+
+HTML_FOOTER: str = "</body></html>"
index 56a41310d78e48a84de4261a7cb106410e8fb7f4..c42bf0c203671debca108cec297643a46c4d50b9 100644 (file)
@@ -15,6 +15,7 @@ from nominatim.typing import SaColumn, SaLabel, SaRow
 from nominatim.api.connection import SearchConnection
 import nominatim.api.types as ntyp
 import nominatim.api.results as nres
+from nominatim.api.logging import log
 
 def _select_column_geometry(column: SaColumn,
                             geometry_output: ntyp.GeometryFormat) -> SaLabel:
@@ -36,6 +37,7 @@ async def find_in_placex(conn: SearchConnection, place: ntyp.PlaceRef,
     """ Search for the given place in the placex table and return the
         base information.
     """
+    log().section("Find in placex table")
     t = conn.t.placex
     sql = sa.select(t.c.place_id, t.c.osm_type, t.c.osm_id, t.c.name,
                     t.c.class_, t.c.type, t.c.admin_level,
@@ -44,8 +46,7 @@ async def find_in_placex(conn: SearchConnection, place: ntyp.PlaceRef,
                     t.c.importance, t.c.wikipedia, t.c.indexed_date,
                     t.c.parent_place_id, t.c.rank_address, t.c.rank_search,
                     t.c.linked_place_id,
-                    sa.func.ST_X(t.c.centroid).label('x'),
-                    sa.func.ST_Y(t.c.centroid).label('y'),
+                    t.c.centroid,
                     _select_column_geometry(t.c.geometry, details.geometry_output))
 
     if isinstance(place, ntyp.PlaceID):
@@ -69,12 +70,12 @@ async def find_in_osmline(conn: SearchConnection, place: ntyp.PlaceRef,
     """ Search for the given place in the osmline table and return the
         base information.
     """
+    log().section("Find in interpolation table")
     t = conn.t.osmline
     sql = sa.select(t.c.place_id, t.c.osm_id, t.c.parent_place_id,
                     t.c.indexed_date, t.c.startnumber, t.c.endnumber,
                     t.c.step, t.c.address, t.c.postcode, t.c.country_code,
-                    sa.func.ST_X(sa.func.ST_Centroid(t.c.linegeo)).label('x'),
-                    sa.func.ST_Y(sa.func.ST_Centroid(t.c.linegeo)).label('y'),
+                    t.c.linegeo.ST_Centroid().label('centroid'),
                     _select_column_geometry(t.c.linegeo, details.geometry_output))
 
     if isinstance(place, ntyp.PlaceID):
@@ -98,12 +99,12 @@ async def find_in_tiger(conn: SearchConnection, place: ntyp.PlaceRef,
     """ Search for the given place in the table of Tiger addresses and return
         the base information. Only lookup by place ID is supported.
     """
+    log().section("Find in TIGER table")
     t = conn.t.tiger
     sql = sa.select(t.c.place_id, t.c.parent_place_id,
                     t.c.startnumber, t.c.endnumber, t.c.step,
                     t.c.postcode,
-                    sa.func.ST_X(sa.func.ST_Centroid(t.c.linegeo)).label('x'),
-                    sa.func.ST_Y(sa.func.ST_Centroid(t.c.linegeo)).label('y'),
+                    t.c.linegeo.ST_Centroid().label('centroid'),
                     _select_column_geometry(t.c.linegeo, details.geometry_output))
 
     if isinstance(place, ntyp.PlaceID):
@@ -119,12 +120,12 @@ async def find_in_postcode(conn: SearchConnection, place: ntyp.PlaceRef,
     """ Search for the given place in the postcode table and return the
         base information. Only lookup by place ID is supported.
     """
+    log().section("Find in postcode table")
     t = conn.t.postcode
     sql = sa.select(t.c.place_id, t.c.parent_place_id,
                     t.c.rank_search, t.c.rank_address,
                     t.c.indexed_date, t.c.postcode, t.c.country_code,
-                    sa.func.ST_X(t.c.geometry).label('x'),
-                    sa.func.ST_Y(t.c.geometry).label('y'),
+                    t.c.geometry.label('centroid'),
                     _select_column_geometry(t.c.geometry, details.geometry_output))
 
     if isinstance(place, ntyp.PlaceID):
@@ -139,30 +140,36 @@ async def get_place_by_id(conn: SearchConnection, place: ntyp.PlaceRef,
                           details: ntyp.LookupDetails) -> Optional[nres.SearchResult]:
     """ Retrieve a place with additional details from the database.
     """
+    log().function('get_place_by_id', place=place, details=details)
+
     if details.geometry_output and details.geometry_output != ntyp.GeometryFormat.GEOJSON:
         raise ValueError("lookup only supports geojosn polygon output.")
 
     row = await find_in_placex(conn, place, details)
     if row is not None:
         result = nres.create_from_placex_row(row)
+        log().var_dump('Result', result)
         await nres.add_result_details(conn, result, details)
         return result
 
     row = await find_in_osmline(conn, place, details)
     if row is not None:
         result = nres.create_from_osmline_row(row)
+        log().var_dump('Result', result)
         await nres.add_result_details(conn, result, details)
         return result
 
     row = await find_in_postcode(conn, place, details)
     if row is not None:
         result = nres.create_from_postcode_row(row)
+        log().var_dump('Result', result)
         await nres.add_result_details(conn, result, details)
         return result
 
     row = await find_in_tiger(conn, place, details)
     if row is not None:
         result = nres.create_from_tiger_row(row)
+        log().var_dump('Result', result)
         await nres.add_result_details(conn, result, details)
         return result
 
index 63c9cf12532974e3977c2199ed92f004b3b90301..10f03393416b33b36f2969e012c0e678d064998a 100644 (file)
@@ -21,6 +21,7 @@ import sqlalchemy as sa
 from nominatim.typing import SaSelect, SaRow
 from nominatim.api.types import Point, LookupDetails
 from nominatim.api.connection import SearchConnection
+from nominatim.api.logging import log
 
 # This file defines complex result data classes.
 # pylint: disable=too-many-instance-attributes
@@ -131,13 +132,6 @@ class SearchResult:
         return self.importance or (0.7500001 - (self.rank_search/40.0))
 
 
-    # pylint: disable=consider-using-f-string
-    def centroid_as_geojson(self) -> str:
-        """ Get the centroid in GeoJSON format.
-        """
-        return '{"type": "Point","coordinates": [%f, %f]}' % self.centroid
-
-
 def _filter_geometries(row: SaRow) -> Dict[str, str]:
     return {k[9:]: v for k, v in row._mapping.items() # pylint: disable=W0212
             if k.startswith('geometry_')}
@@ -165,7 +159,7 @@ def create_from_placex_row(row: SaRow) -> SearchResult:
                         importance=row.importance,
                         country_code=row.country_code,
                         indexed_date=getattr(row, 'indexed_date'),
-                        centroid=Point(row.x, row.y),
+                        centroid=Point.from_wkb(row.centroid.data),
                         geometry=_filter_geometries(row))
 
 
@@ -185,7 +179,7 @@ def create_from_osmline_row(row: SaRow) -> SearchResult:
                                    'step': str(row.step)},
                         country_code=row.country_code,
                         indexed_date=getattr(row, 'indexed_date'),
-                        centroid=Point(row.x, row.y),
+                        centroid=Point.from_wkb(row.centroid.data),
                         geometry=_filter_geometries(row))
 
 
@@ -202,7 +196,7 @@ def create_from_tiger_row(row: SaRow) -> SearchResult:
                                    'endnumber': str(row.endnumber),
                                    'step': str(row.step)},
                         country_code='us',
-                        centroid=Point(row.x, row.y),
+                        centroid=Point.from_wkb(row.centroid.data),
                         geometry=_filter_geometries(row))
 
 
@@ -218,7 +212,7 @@ def create_from_postcode_row(row: SaRow) -> SearchResult:
                         rank_search=row.rank_search,
                         rank_address=row.rank_address,
                         country_code=row.country_code,
-                        centroid=Point(row.x, row.y),
+                        centroid=Point.from_wkb(row.centroid.data),
                         indexed_date=row.indexed_date,
                         geometry=_filter_geometries(row))
 
@@ -228,13 +222,18 @@ async def add_result_details(conn: SearchConnection, result: SearchResult,
     """ Retrieve more details from the database according to the
         parameters specified in 'details'.
     """
+    log().section('Query details for result')
     if details.address_details:
+        log().comment('Query address details')
         await complete_address_details(conn, result)
     if details.linked_places:
+        log().comment('Query linked places')
         await complete_linked_places(conn, result)
     if details.parented_places:
+        log().comment('Query parent places')
         await complete_parented_places(conn, result)
     if details.keywords:
+        log().comment('Query keywords')
         await complete_keywords(conn, result)
 
 
index 89b8111107fff712908fe70cad48d61404263fd0..9dc3ff2e6341cfcdbbc34f28e2c10b280797ea34 100644 (file)
@@ -10,6 +10,7 @@ Complex datatypes used by the Nominatim API.
 from typing import Optional, Union, NamedTuple
 import dataclasses
 import enum
+from struct import unpack
 
 @dataclasses.dataclass
 class PlaceID:
@@ -55,6 +56,33 @@ class Point(NamedTuple):
         return self.x
 
 
+    def to_geojson(self) -> str:
+        """ Return the point in GeoJSON format.
+        """
+        return f'{{"type": "Point","coordinates": [{self.x}, {self.y}]}}'
+
+
+    @staticmethod
+    def from_wkb(wkb: bytes) -> 'Point':
+        """ Create a point from EWKB as returned from the database.
+        """
+        if len(wkb) != 25:
+            raise ValueError("Point wkb has unexpected length")
+        if wkb[0] == 0:
+            gtype, srid, x, y = unpack('>iidd', wkb[1:])
+        elif wkb[0] == 1:
+            gtype, srid, x, y = unpack('<iidd', wkb[1:])
+        else:
+            raise ValueError("WKB has unknown endian value.")
+
+        if gtype != 0x20000001:
+            raise ValueError("WKB must be a point geometry.")
+        if srid != 4326:
+            raise ValueError("Only WGS84 WKB supported.")
+
+        return Point(x, y)
+
+
 class GeometryFormat(enum.Flag):
     """ Geometry output formats supported by Nominatim.
     """
index 7c8ba80897ff3cec093d355a71881638291de8bb..3f26f903c5675f28502cef19641aa86f34f88dbc 100644 (file)
@@ -96,7 +96,7 @@ def _add_parent_rows_grouped(writer: JsonWriter, rows: napi.AddressLines,
 def _format_search_json(result: napi.SearchResult, options: Mapping[str, Any]) -> str:
     locales = options.get('locales', napi.Locales())
     geom = result.geometry.get('geojson')
-    centroid = result.centroid_as_geojson()
+    centroid = result.centroid.to_geojson()
 
     out = JsonWriter()
     out.start_object()\
index 8aa28cfe66d8a26e459a07b2e0a8c263dba2eeb7..35028526aba6983ed5d7ed221b593fe52caa4cda 100644 (file)
@@ -13,12 +13,14 @@ import abc
 
 from nominatim.config import Configuration
 import nominatim.api as napi
+import nominatim.api.logging as loglib
 from nominatim.api.v1.format import dispatch as formatting
 
 CONTENT_TYPE = {
   'text': 'text/plain; charset=utf-8',
   'xml': 'text/xml; charset=utf-8',
-  'jsonp': 'application/javascript'
+  'jsonp': 'application/javascript',
+  'debug': 'text/html; charset=utf-8'
 }
 
 
@@ -131,6 +133,18 @@ class ASGIAdaptor(abc.ABC):
                or self.config().DEFAULT_LANGUAGE
 
 
+    def setup_debugging(self) -> bool:
+        """ Set up collection of debug information if requested.
+
+            Return True when debugging was requested.
+        """
+        if self.get_bool('debug', False):
+            loglib.set_log_output('html')
+            return True
+
+        return False
+
+
 def parse_format(params: ASGIAdaptor, result_type: Type[Any], default: str) -> str:
     """ Get and check the 'format' parameter and prepare the formatter.
         `fmtter` is a formatter and `default` the
@@ -175,6 +189,8 @@ async def details_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) ->
             raise params.error("Missing ID parameter 'place_id' or 'osmtype'.")
         place = napi.OsmID(osmtype, params.get_int('osmid'), params.get('class'))
 
+    debug = params.setup_debugging()
+
     details = napi.LookupDetails(address_details=params.get_bool('addressdetails', False),
                                  linked_places=params.get_bool('linkedplaces', False),
                                  parented_places=params.get_bool('hierarchy', False),
@@ -184,10 +200,12 @@ async def details_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) ->
         details.geometry_output = napi.GeometryFormat.GEOJSON
 
     locales = napi.Locales.from_accept_languages(params.get_accepted_languages())
-    print(locales.languages)
 
     result = await api.lookup(place, details)
 
+    if debug:
+        return params.build_response(loglib.get_and_disable(), 'debug')
+
     if result is None:
         raise params.error('No place with that OSM ID found.', status=404)
 
index 7d117a8c7f495209703c25d9b8a1364d8725515c..0c88493bfbd34567f2d6ec4fb9d02deca0217065 100644 (file)
@@ -48,7 +48,8 @@ def migrate(config: Configuration, paths: Any) -> int:
 
         has_run_migration = False
         for version, func in _MIGRATION_FUNCTIONS:
-            if db_version <= version:
+            if db_version < version or \
+               (db_version == (3, 5, 0, 99) and version == (3, 5, 0, 99)):
                 title = func.__doc__ or ''
                 LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], version)
                 kwargs = dict(conn=conn, config=config, paths=paths)
@@ -371,3 +372,16 @@ def enable_forward_dependencies(conn: Connection, **_: Any) -> None:
                                  ON planet_osm_rels USING gin (parts)
                                  WITH (fastupdate=off)""")
                 cur.execute("ANALYZE planet_osm_ways")
+
+
+@_migration(4, 2, 99, 1)
+def add_improved_geometry_reverse_placenode_index(conn: Connection, **_: Any) -> None:
+    """ Create improved index for reverse lookup of place nodes.
+    """
+    with conn.cursor() as cur:
+        cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPlaceNode
+                       ON placex
+                       USING gist (ST_Buffer(geometry, reverse_place_diameter(rank_search)))
+                       WHERE rank_address between 4 and 25 AND type != 'postcode'
+                         AND name is not null AND linked_place_id is null AND osm_type = 'N'
+                    """)
index 40e3bda42d9f6f2a06fad2bf8a588b803d98c9b8..346af5eb651f969bd8bd5694c34df4bc0840171e 100644 (file)
@@ -34,7 +34,7 @@ class NominatimVersion(NamedTuple):
         return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}"
 
 
-NOMINATIM_VERSION = NominatimVersion(4, 2, 99, 0)
+NOMINATIM_VERSION = NominatimVersion(4, 2, 99, 1)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)
index 294240163140ec1d1e272b267d37034a28615d9e..0275e275e96e24cda90c6894de7a6ca99fb75af8 100644 (file)
@@ -14,6 +14,7 @@ import datetime as dt
 
 import nominatim.api as napi
 from nominatim.db.sql_preprocessor import SQLPreprocessor
+import nominatim.api.logging as loglib
 
 class APITester:
 
@@ -138,6 +139,8 @@ def apiobj(temp_db_with_extensions, temp_db_conn, monkeypatch):
     SQLPreprocessor(temp_db_conn, testapi.api.config)\
         .run_sql_file(temp_db_conn, 'functions/address_lookup.sql')
 
+    loglib.set_log_output('text')
     yield testapi
+    print(loglib.get_and_disable())
 
     testapi.api.close()
index 966059c480ddf627cc3989c9a835dea8f40d0995..0b5dccfb63d36d5075a47e167d4a16e273af6259 100644 (file)
@@ -80,7 +80,7 @@ class TestCliDetailsCall:
     @pytest.fixture(autouse=True)
     def setup_status_mock(self, monkeypatch):
         result = napi.SearchResult(napi.SourceTable.PLACEX, ('place', 'thing'),
-                                   (1.0, -3.0))
+                                   napi.Point(1.0, -3.0))
 
         monkeypatch.setattr(napi.NominatimAPI, 'lookup',
                             lambda *args: result)
@@ -90,7 +90,7 @@ class TestCliDetailsCall:
                                         ('--relation', '1'),
                                         ('--place_id', '10001')])
 
-    def test_status_json_format(self, cli_call, tmp_path, capsys, params):
+    def test_details_json_format(self, cli_call, tmp_path, capsys, params):
         result = cli_call('details', '--project-dir', str(tmp_path), *params)
 
         assert result == 0