nominatim/api/search/db_searches.py

   1 # SPDX-License-Identifier: GPL-3.0-or-later
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2023 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Implementation of the acutal database accesses for forward search.
   9 """
  10 from typing import List, Tuple, AsyncIterator, Dict, Any, Callable
  11 import abc
  12
  13 import sqlalchemy as sa
  14 from sqlalchemy.dialects.postgresql import array_agg
  15
  16 from nominatim.typing import SaFromClause, SaScalarSelect, SaColumn, \
  17                              SaExpression, SaSelect, SaLambdaSelect, SaRow, SaBind
  18 from nominatim.api.connection import SearchConnection
  19 from nominatim.api.types import SearchDetails, DataLayer, GeometryFormat, Bbox
  20 import nominatim.api.results as nres
  21 from nominatim.api.search.db_search_fields import SearchData, WeightedCategories
  22 from nominatim.db.sqlalchemy_types import Geometry
  23
  24 #pylint: disable=singleton-comparison,not-callable
  25 #pylint: disable=too-many-branches,too-many-arguments,too-many-locals,too-many-statements
  26
  27 def no_index(expr: SaColumn) -> SaColumn:
  28     """ Wrap the given expression, so that the query planner will
  29         refrain from using the expression for index lookup.
  30     """
  31     return sa.func.coalesce(sa.null(), expr) # pylint: disable=not-callable
  32
  33
  34 def _details_to_bind_params(details: SearchDetails) -> Dict[str, Any]:
  35     """ Create a dictionary from search parameters that can be used
  36         as bind parameter for SQL execute.
  37     """
  38     return {'limit': details.max_results,
  39             'min_rank': details.min_rank,
  40             'max_rank': details.max_rank,
  41             'viewbox': details.viewbox,
  42             'viewbox2': details.viewbox_x2,
  43             'near': details.near,
  44             'near_radius': details.near_radius,
  45             'excluded': details.excluded,
  46             'countries': details.countries}
  47
  48
  49 LIMIT_PARAM: SaBind = sa.bindparam('limit')
  50 MIN_RANK_PARAM: SaBind = sa.bindparam('min_rank')
  51 MAX_RANK_PARAM: SaBind = sa.bindparam('max_rank')
  52 VIEWBOX_PARAM: SaBind = sa.bindparam('viewbox', type_=Geometry)
  53 VIEWBOX2_PARAM: SaBind = sa.bindparam('viewbox2', type_=Geometry)
  54 NEAR_PARAM: SaBind = sa.bindparam('near', type_=Geometry)
  55 NEAR_RADIUS_PARAM: SaBind = sa.bindparam('near_radius')
  56 COUNTRIES_PARAM: SaBind = sa.bindparam('countries')
  57
  58 def _within_near(t: SaFromClause) -> Callable[[], SaExpression]:
  59     return lambda: t.c.geometry.ST_DWithin(NEAR_PARAM, NEAR_RADIUS_PARAM)
  60
  61 def _exclude_places(t: SaFromClause) -> Callable[[], SaExpression]:
  62     return lambda: t.c.place_id.not_in(sa.bindparam('excluded'))
  63
  64 def _select_placex(t: SaFromClause) -> SaSelect:
  65     return sa.select(t.c.place_id, t.c.osm_type, t.c.osm_id, t.c.name,
  66                      t.c.class_, t.c.type,
  67                      t.c.address, t.c.extratags,
  68                      t.c.housenumber, t.c.postcode, t.c.country_code,
  69                      t.c.wikipedia,
  70                      t.c.parent_place_id, t.c.rank_address, t.c.rank_search,
  71                      t.c.linked_place_id, t.c.admin_level,
  72                      t.c.centroid,
  73                      t.c.geometry.ST_Expand(0).label('bbox'))
  74
  75
  76 def _add_geometry_columns(sql: SaLambdaSelect, col: SaColumn, details: SearchDetails) -> SaSelect:
  77     out = []
  78
  79     if details.geometry_simplification > 0.0:
  80         col = sa.func.ST_SimplifyPreserveTopology(col, details.geometry_simplification)
  81
  82     if details.geometry_output & GeometryFormat.GEOJSON:
  83         out.append(sa.func.ST_AsGeoJSON(col, 7).label('geometry_geojson'))
  84     if details.geometry_output & GeometryFormat.TEXT:
  85         out.append(sa.func.ST_AsText(col).label('geometry_text'))
  86     if details.geometry_output & GeometryFormat.KML:
  87         out.append(sa.func.ST_AsKML(col, 7).label('geometry_kml'))
  88     if details.geometry_output & GeometryFormat.SVG:
  89         out.append(sa.func.ST_AsSVG(col, 0, 7).label('geometry_svg'))
  90
  91     return sql.add_columns(*out)
  92
  93
  94 def _make_interpolation_subquery(table: SaFromClause, inner: SaFromClause,
  95                                  numerals: List[int], details: SearchDetails) -> SaScalarSelect:
  96     all_ids = array_agg(table.c.place_id) # type: ignore[no-untyped-call]
  97     sql = sa.select(all_ids).where(table.c.parent_place_id == inner.c.place_id)
  98
  99     if len(numerals) == 1:
 100         sql = sql.where(sa.between(numerals[0], table.c.startnumber, table.c.endnumber))\
 101                  .where((numerals[0] - table.c.startnumber) % table.c.step == 0)
 102     else:
 103         sql = sql.where(sa.or_(
 104                 *(sa.and_(sa.between(n, table.c.startnumber, table.c.endnumber),
 105                           (n - table.c.startnumber) % table.c.step == 0)
 106                   for n in numerals)))
 107
 108     if details.excluded:
 109         sql = sql.where(_exclude_places(table))
 110
 111     return sql.scalar_subquery()
 112
 113
 114 def _filter_by_layer(table: SaFromClause, layers: DataLayer) -> SaColumn:
 115     orexpr: List[SaExpression] = []
 116     if layers & DataLayer.ADDRESS and layers & DataLayer.POI:
 117         orexpr.append(no_index(table.c.rank_address).between(1, 30))
 118     elif layers & DataLayer.ADDRESS:
 119         orexpr.append(no_index(table.c.rank_address).between(1, 29))
 120         orexpr.append(sa.and_(no_index(table.c.rank_address) == 30,
 121                               sa.or_(table.c.housenumber != None,
 122                                      table.c.address.has_key('addr:housename'))))
 123     elif layers & DataLayer.POI:
 124         orexpr.append(sa.and_(no_index(table.c.rank_address) == 30,
 125                               table.c.class_.not_in(('place', 'building'))))
 126
 127     if layers & DataLayer.MANMADE:
 128         exclude = []
 129         if not layers & DataLayer.RAILWAY:
 130             exclude.append('railway')
 131         if not layers & DataLayer.NATURAL:
 132             exclude.extend(('natural', 'water', 'waterway'))
 133         orexpr.append(sa.and_(table.c.class_.not_in(tuple(exclude)),
 134                               no_index(table.c.rank_address) == 0))
 135     else:
 136         include = []
 137         if layers & DataLayer.RAILWAY:
 138             include.append('railway')
 139         if layers & DataLayer.NATURAL:
 140             include.extend(('natural', 'water', 'waterway'))
 141         orexpr.append(sa.and_(table.c.class_.in_(tuple(include)),
 142                               no_index(table.c.rank_address) == 0))
 143
 144     if len(orexpr) == 1:
 145         return orexpr[0]
 146
 147     return sa.or_(*orexpr)
 148
 149
 150 def _interpolated_position(table: SaFromClause, nr: SaColumn) -> SaColumn:
 151     pos = sa.cast(nr - table.c.startnumber, sa.Float) / (table.c.endnumber - table.c.startnumber)
 152     return sa.case(
 153             (table.c.endnumber == table.c.startnumber, table.c.linegeo.ST_Centroid()),
 154             else_=table.c.linegeo.ST_LineInterpolatePoint(pos)).label('centroid')
 155
 156
 157 async def _get_placex_housenumbers(conn: SearchConnection,
 158                                    place_ids: List[int],
 159                                    details: SearchDetails) -> AsyncIterator[nres.SearchResult]:
 160     t = conn.t.placex
 161     sql = _select_placex(t).add_columns(t.c.importance)\
 162                            .where(t.c.place_id.in_(place_ids))
 163
 164     if details.geometry_output:
 165         sql = _add_geometry_columns(sql, t.c.geometry, details)
 166
 167     for row in await conn.execute(sql):
 168         result = nres.create_from_placex_row(row, nres.SearchResult)
 169         assert result
 170         result.bbox = Bbox.from_wkb(row.bbox)
 171         yield result
 172
 173
 174 async def _get_osmline(conn: SearchConnection, place_ids: List[int],
 175                        numerals: List[int],
 176                        details: SearchDetails) -> AsyncIterator[nres.SearchResult]:
 177     t = conn.t.osmline
 178     values = sa.values(sa.Column('nr', sa.Integer()), name='housenumber')\
 179                .data([(n,) for n in numerals])
 180     sql = sa.select(t.c.place_id, t.c.osm_id,
 181                     t.c.parent_place_id, t.c.address,
 182                     values.c.nr.label('housenumber'),
 183                     _interpolated_position(t, values.c.nr),
 184                     t.c.postcode, t.c.country_code)\
 185             .where(t.c.place_id.in_(place_ids))\
 186             .join(values, values.c.nr.between(t.c.startnumber, t.c.endnumber))
 187
 188     if details.geometry_output:
 189         sub = sql.subquery()
 190         sql = _add_geometry_columns(sa.select(sub), sub.c.centroid, details)
 191
 192     for row in await conn.execute(sql):
 193         result = nres.create_from_osmline_row(row, nres.SearchResult)
 194         assert result
 195         yield result
 196
 197
 198 async def _get_tiger(conn: SearchConnection, place_ids: List[int],
 199                      numerals: List[int], osm_id: int,
 200                      details: SearchDetails) -> AsyncIterator[nres.SearchResult]:
 201     t = conn.t.tiger
 202     values = sa.values(sa.Column('nr', sa.Integer()), name='housenumber')\
 203                .data([(n,) for n in numerals])
 204     sql = sa.select(t.c.place_id, t.c.parent_place_id,
 205                     sa.literal('W').label('osm_type'),
 206                     sa.literal(osm_id).label('osm_id'),
 207                     values.c.nr.label('housenumber'),
 208                     _interpolated_position(t, values.c.nr),
 209                     t.c.postcode)\
 210             .where(t.c.place_id.in_(place_ids))\
 211             .join(values, values.c.nr.between(t.c.startnumber, t.c.endnumber))
 212
 213     if details.geometry_output:
 214         sub = sql.subquery()
 215         sql = _add_geometry_columns(sa.select(sub), sub.c.centroid, details)
 216
 217     for row in await conn.execute(sql):
 218         result = nres.create_from_tiger_row(row, nres.SearchResult)
 219         assert result
 220         yield result
 221
 222
 223 class AbstractSearch(abc.ABC):
 224     """ Encapuslation of a single lookup in the database.
 225     """
 226
 227     def __init__(self, penalty: float) -> None:
 228         self.penalty = penalty
 229
 230     @abc.abstractmethod
 231     async def lookup(self, conn: SearchConnection,
 232                      details: SearchDetails) -> nres.SearchResults:
 233         """ Find results for the search in the database.
 234         """
 235
 236
 237 class NearSearch(AbstractSearch):
 238     """ Category search of a place type near the result of another search.
 239     """
 240     def __init__(self, penalty: float, categories: WeightedCategories,
 241                  search: AbstractSearch) -> None:
 242         super().__init__(penalty)
 243         self.search = search
 244         self.categories = categories
 245
 246
 247     async def lookup(self, conn: SearchConnection,
 248                      details: SearchDetails) -> nres.SearchResults:
 249         """ Find results for the search in the database.
 250         """
 251         results = nres.SearchResults()
 252         base = await self.search.lookup(conn, details)
 253
 254         if not base:
 255             return results
 256
 257         base.sort(key=lambda r: (r.accuracy, r.rank_search))
 258         max_accuracy = base[0].accuracy + 0.5
 259         if base[0].rank_address == 0:
 260             min_rank = 0
 261             max_rank = 0
 262         elif base[0].rank_address < 26:
 263             min_rank = 1
 264             max_rank = min(25, base[0].rank_address + 4)
 265         else:
 266             min_rank = 26
 267             max_rank = 30
 268         base = nres.SearchResults(r for r in base if r.source_table == nres.SourceTable.PLACEX
 269                                                      and r.accuracy <= max_accuracy
 270                                                      and r.bbox and r.bbox.area < 20
 271                                                      and r.rank_address >= min_rank
 272                                                      and r.rank_address <= max_rank)
 273
 274         if base:
 275             baseids = [b.place_id for b in base[:5] if b.place_id]
 276
 277             for category, penalty in self.categories:
 278                 await self.lookup_category(results, conn, baseids, category, penalty, details)
 279                 if len(results) >= details.max_results:
 280                     break
 281
 282         return results
 283
 284
 285     async def lookup_category(self, results: nres.SearchResults,
 286                               conn: SearchConnection, ids: List[int],
 287                               category: Tuple[str, str], penalty: float,
 288                               details: SearchDetails) -> None:
 289         """ Find places of the given category near the list of
 290             place ids and add the results to 'results'.
 291         """
 292         table = await conn.get_class_table(*category)
 293
 294         tgeom = conn.t.placex.alias('pgeom')
 295
 296         if table is None:
 297             # No classtype table available, do a simplified lookup in placex.
 298             table = conn.t.placex.alias('inner')
 299             sql = sa.select(table.c.place_id,
 300                             sa.func.min(tgeom.c.centroid.ST_Distance(table.c.centroid))
 301                               .label('dist'))\
 302                     .join(tgeom, table.c.geometry.intersects(tgeom.c.centroid.ST_Expand(0.01)))\
 303                     .where(table.c.class_ == category[0])\
 304                     .where(table.c.type == category[1])
 305         else:
 306             # Use classtype table. We can afford to use a larger
 307             # radius for the lookup.
 308             sql = sa.select(table.c.place_id,
 309                             sa.func.min(tgeom.c.centroid.ST_Distance(table.c.centroid))
 310                               .label('dist'))\
 311                     .join(tgeom,
 312                           table.c.centroid.ST_CoveredBy(
 313                               sa.case((sa.and_(tgeom.c.rank_address > 9,
 314                                                 tgeom.c.geometry.is_area()),
 315                                        tgeom.c.geometry),
 316                                       else_ = tgeom.c.centroid.ST_Expand(0.05))))
 317
 318         inner = sql.where(tgeom.c.place_id.in_(ids))\
 319                    .group_by(table.c.place_id).subquery()
 320
 321         t = conn.t.placex
 322         sql = _select_placex(t).add_columns((-inner.c.dist).label('importance'))\
 323                                .join(inner, inner.c.place_id == t.c.place_id)\
 324                                .order_by(inner.c.dist)
 325
 326         sql = sql.where(no_index(t.c.rank_address).between(MIN_RANK_PARAM, MAX_RANK_PARAM))
 327         if details.countries:
 328             sql = sql.where(t.c.country_code.in_(COUNTRIES_PARAM))
 329         if details.excluded:
 330             sql = sql.where(_exclude_places(t))
 331         if details.layers is not None:
 332             sql = sql.where(_filter_by_layer(t, details.layers))
 333
 334         sql = sql.limit(LIMIT_PARAM)
 335         for row in await conn.execute(sql, _details_to_bind_params(details)):
 336             result = nres.create_from_placex_row(row, nres.SearchResult)
 337             assert result
 338             result.accuracy = self.penalty + penalty
 339             result.bbox = Bbox.from_wkb(row.bbox)
 340             results.append(result)
 341
 342
 343
 344 class PoiSearch(AbstractSearch):
 345     """ Category search in a geographic area.
 346     """
 347     def __init__(self, sdata: SearchData) -> None:
 348         super().__init__(sdata.penalty)
 349         self.qualifiers = sdata.qualifiers
 350         self.countries = sdata.countries
 351
 352
 353     async def lookup(self, conn: SearchConnection,
 354                      details: SearchDetails) -> nres.SearchResults:
 355         """ Find results for the search in the database.
 356         """
 357         bind_params = _details_to_bind_params(details)
 358         t = conn.t.placex
 359
 360         rows: List[SaRow] = []
 361
 362         if details.near and details.near_radius is not None and details.near_radius < 0.2:
 363             # simply search in placex table
 364             def _base_query() -> SaSelect:
 365                 return _select_placex(t) \
 366                            .add_columns((-t.c.centroid.ST_Distance(NEAR_PARAM))
 367                                          .label('importance'))\
 368                            .where(t.c.linked_place_id == None) \
 369                            .where(t.c.geometry.ST_DWithin(NEAR_PARAM, NEAR_RADIUS_PARAM)) \
 370                            .order_by(t.c.centroid.ST_Distance(NEAR_PARAM)) \
 371                            .limit(LIMIT_PARAM)
 372
 373             classtype = self.qualifiers.values
 374             if len(classtype) == 1:
 375                 cclass, ctype = classtype[0]
 376                 sql: SaLambdaSelect = sa.lambda_stmt(lambda: _base_query()
 377                                                  .where(t.c.class_ == cclass)
 378                                                  .where(t.c.type == ctype))
 379             else:
 380                 sql = _base_query().where(sa.or_(*(sa.and_(t.c.class_ == cls, t.c.type == typ)
 381                                                    for cls, typ in classtype)))
 382
 383             if self.countries:
 384                 sql = sql.where(t.c.country_code.in_(self.countries.values))
 385
 386             if details.viewbox is not None and details.bounded_viewbox:
 387                 sql = sql.where(t.c.geometry.intersects(VIEWBOX_PARAM))
 388
 389             rows.extend(await conn.execute(sql, bind_params))
 390         else:
 391             # use the class type tables
 392             for category in self.qualifiers.values:
 393                 table = await conn.get_class_table(*category)
 394                 if table is not None:
 395                     sql = _select_placex(t)\
 396                                .add_columns(t.c.importance)\
 397                                .join(table, t.c.place_id == table.c.place_id)\
 398                                .where(t.c.class_ == category[0])\
 399                                .where(t.c.type == category[1])
 400
 401                     if details.viewbox is not None and details.bounded_viewbox:
 402                         sql = sql.where(table.c.centroid.intersects(VIEWBOX_PARAM))
 403
 404                     if details.near and details.near_radius is not None:
 405                         sql = sql.order_by(table.c.centroid.ST_Distance(NEAR_PARAM))\
 406                                  .where(table.c.centroid.ST_DWithin(NEAR_PARAM,
 407                                                                     NEAR_RADIUS_PARAM))
 408
 409                     if self.countries:
 410                         sql = sql.where(t.c.country_code.in_(self.countries.values))
 411
 412                     sql = sql.limit(LIMIT_PARAM)
 413                     rows.extend(await conn.execute(sql, bind_params))
 414
 415         results = nres.SearchResults()
 416         for row in rows:
 417             result = nres.create_from_placex_row(row, nres.SearchResult)
 418             assert result
 419             result.accuracy = self.penalty + self.qualifiers.get_penalty((row.class_, row.type))
 420             result.bbox = Bbox.from_wkb(row.bbox)
 421             results.append(result)
 422
 423         return results
 424
 425
 426 class CountrySearch(AbstractSearch):
 427     """ Search for a country name or country code.
 428     """
 429     def __init__(self, sdata: SearchData) -> None:
 430         super().__init__(sdata.penalty)
 431         self.countries = sdata.countries
 432
 433
 434     async def lookup(self, conn: SearchConnection,
 435                      details: SearchDetails) -> nres.SearchResults:
 436         """ Find results for the search in the database.
 437         """
 438         t = conn.t.placex
 439
 440         ccodes = self.countries.values
 441         sql = _select_placex(t)\
 442                 .add_columns(t.c.importance)\
 443                 .where(t.c.country_code.in_(ccodes))\
 444                 .where(t.c.rank_address == 4)
 445
 446         if details.geometry_output:
 447             sql = _add_geometry_columns(sql, t.c.geometry, details)
 448
 449         if details.excluded:
 450             sql = sql.where(_exclude_places(t))
 451
 452         if details.viewbox is not None and details.bounded_viewbox:
 453             sql = sql.where(lambda: t.c.geometry.intersects(VIEWBOX_PARAM))
 454
 455         if details.near is not None and details.near_radius is not None:
 456             sql = sql.where(_within_near(t))
 457
 458         results = nres.SearchResults()
 459         for row in await conn.execute(sql, _details_to_bind_params(details)):
 460             result = nres.create_from_placex_row(row, nres.SearchResult)
 461             assert result
 462             result.accuracy = self.penalty + self.countries.get_penalty(row.country_code, 5.0)
 463             result.bbox = Bbox.from_wkb(row.bbox)
 464             results.append(result)
 465
 466         return results or await self.lookup_in_country_table(conn, details)
 467
 468
 469     async def lookup_in_country_table(self, conn: SearchConnection,
 470                                       details: SearchDetails) -> nres.SearchResults:
 471         """ Look up the country in the fallback country tables.
 472         """
 473         # Avoid the fallback search when this is a more search. Country results
 474         # usually are in the first batch of results and it is not possible
 475         # to exclude these fallbacks.
 476         if details.excluded:
 477             return nres.SearchResults()
 478
 479         t = conn.t.country_name
 480         tgrid = conn.t.country_grid
 481
 482         sql = sa.select(tgrid.c.country_code,
 483                         tgrid.c.geometry.ST_Centroid().ST_Collect().ST_Centroid()
 484                               .label('centroid'),
 485                         tgrid.c.geometry.ST_Collect().ST_Expand(0).label('bbox'))\
 486                 .where(tgrid.c.country_code.in_(self.countries.values))\
 487                 .group_by(tgrid.c.country_code)
 488
 489         if details.viewbox is not None and details.bounded_viewbox:
 490             sql = sql.where(tgrid.c.geometry.intersects(VIEWBOX_PARAM))
 491         if details.near is not None and details.near_radius is not None:
 492             sql = sql.where(_within_near(tgrid))
 493
 494         sub = sql.subquery('grid')
 495
 496         sql = sa.select(t.c.country_code,
 497                         t.c.name.merge(t.c.derived_name).label('name'),
 498                         sub.c.centroid, sub.c.bbox)\
 499                 .join(sub, t.c.country_code == sub.c.country_code)
 500
 501         if details.geometry_output:
 502             sql = _add_geometry_columns(sql, sub.c.centroid, details)
 503
 504         results = nres.SearchResults()
 505         for row in await conn.execute(sql, _details_to_bind_params(details)):
 506             result = nres.create_from_country_row(row, nres.SearchResult)
 507             assert result
 508             result.bbox = Bbox.from_wkb(row.bbox)
 509             result.accuracy = self.penalty + self.countries.get_penalty(row.country_code, 5.0)
 510             results.append(result)
 511
 512         return results
 513
 514
 515
 516 class PostcodeSearch(AbstractSearch):
 517     """ Search for a postcode.
 518     """
 519     def __init__(self, extra_penalty: float, sdata: SearchData) -> None:
 520         super().__init__(sdata.penalty + extra_penalty)
 521         self.countries = sdata.countries
 522         self.postcodes = sdata.postcodes
 523         self.lookups = sdata.lookups
 524         self.rankings = sdata.rankings
 525
 526
 527     async def lookup(self, conn: SearchConnection,
 528                      details: SearchDetails) -> nres.SearchResults:
 529         """ Find results for the search in the database.
 530         """
 531         t = conn.t.postcode
 532         pcs = self.postcodes.values
 533
 534         sql = sa.select(t.c.place_id, t.c.parent_place_id,
 535                         t.c.rank_search, t.c.rank_address,
 536                         t.c.postcode, t.c.country_code,
 537                         t.c.geometry.label('centroid'))\
 538                 .where(t.c.postcode.in_(pcs))
 539
 540         if details.geometry_output:
 541             sql = _add_geometry_columns(sql, t.c.geometry, details)
 542
 543         penalty: SaExpression = sa.literal(self.penalty)
 544
 545         if details.viewbox is not None:
 546             if details.bounded_viewbox:
 547                 sql = sql.where(t.c.geometry.intersects(VIEWBOX_PARAM))
 548             else:
 549                 penalty += sa.case((t.c.geometry.intersects(VIEWBOX_PARAM), 0.0),
 550                                    (t.c.geometry.intersects(VIEWBOX2_PARAM), 0.5),
 551                                    else_=1.0)
 552
 553         if details.near is not None:
 554             if details.near_radius is not None:
 555                 sql = sql.where(_within_near(t))
 556             sql = sql.order_by(t.c.geometry.ST_Distance(NEAR_PARAM))
 557
 558         if self.countries:
 559             sql = sql.where(t.c.country_code.in_(self.countries.values))
 560
 561         if details.excluded:
 562             sql = sql.where(_exclude_places(t))
 563
 564         if self.lookups:
 565             assert len(self.lookups) == 1
 566             assert self.lookups[0].lookup_type == 'restrict'
 567             tsearch = conn.t.search_name
 568             sql = sql.where(tsearch.c.place_id == t.c.parent_place_id)\
 569                      .where((tsearch.c.name_vector + tsearch.c.nameaddress_vector)
 570                                      .contains(self.lookups[0].tokens))
 571
 572         for ranking in self.rankings:
 573             penalty += ranking.sql_penalty(conn.t.search_name)
 574         penalty += sa.case(*((t.c.postcode == v, p) for v, p in self.postcodes),
 575                        else_=1.0)
 576
 577
 578         sql = sql.add_columns(penalty.label('accuracy'))
 579         sql = sql.order_by('accuracy').limit(LIMIT_PARAM)
 580
 581         results = nres.SearchResults()
 582         for row in await conn.execute(sql, _details_to_bind_params(details)):
 583             result = nres.create_from_postcode_row(row, nres.SearchResult)
 584             assert result
 585             result.accuracy = row.accuracy
 586             results.append(result)
 587
 588         return results
 589
 590
 591
 592 class PlaceSearch(AbstractSearch):
 593     """ Generic search for an address or named place.
 594     """
 595     def __init__(self, extra_penalty: float, sdata: SearchData, expected_count: int) -> None:
 596         super().__init__(sdata.penalty + extra_penalty)
 597         self.countries = sdata.countries
 598         self.postcodes = sdata.postcodes
 599         self.housenumbers = sdata.housenumbers
 600         self.qualifiers = sdata.qualifiers
 601         self.lookups = sdata.lookups
 602         self.rankings = sdata.rankings
 603         self.expected_count = expected_count
 604
 605
 606     async def lookup(self, conn: SearchConnection,
 607                      details: SearchDetails) -> nres.SearchResults:
 608         """ Find results for the search in the database.
 609         """
 610         t = conn.t.placex
 611         tsearch = conn.t.search_name
 612
 613         sql: SaLambdaSelect = sa.lambda_stmt(lambda:
 614                   _select_placex(t).where(t.c.place_id == tsearch.c.place_id))
 615
 616
 617         if details.geometry_output:
 618             sql = _add_geometry_columns(sql, t.c.geometry, details)
 619
 620         penalty: SaExpression = sa.literal(self.penalty)
 621         for ranking in self.rankings:
 622             penalty += ranking.sql_penalty(tsearch)
 623
 624         for lookup in self.lookups:
 625             sql = sql.where(lookup.sql_condition(tsearch))
 626
 627         if self.countries:
 628             sql = sql.where(tsearch.c.country_code.in_(self.countries.values))
 629
 630         if self.postcodes:
 631             # if a postcode is given, don't search for state or country level objects
 632             sql = sql.where(tsearch.c.address_rank > 9)
 633             tpc = conn.t.postcode
 634             pcs = self.postcodes.values
 635             if self.expected_count > 1000:
 636                 # Many results expected. Restrict by postcode.
 637                 sql = sql.where(sa.select(tpc.c.postcode)
 638                                   .where(tpc.c.postcode.in_(pcs))
 639                                   .where(tsearch.c.centroid.ST_DWithin(tpc.c.geometry, 0.12))
 640                                   .exists())
 641
 642             # Less results, only have a preference for close postcodes
 643             pc_near = sa.select(sa.func.min(tpc.c.geometry.ST_Distance(tsearch.c.centroid)))\
 644                       .where(tpc.c.postcode.in_(pcs))\
 645                       .scalar_subquery()
 646             penalty += sa.case((t.c.postcode.in_(pcs), 0.0),
 647                                else_=sa.func.coalesce(pc_near, 2.0))
 648
 649         if details.viewbox is not None:
 650             if details.bounded_viewbox:
 651                 if details.viewbox.area < 0.2:
 652                     sql = sql.where(tsearch.c.centroid.intersects(VIEWBOX_PARAM))
 653                 else:
 654                     sql = sql.where(tsearch.c.centroid.ST_Intersects_no_index(VIEWBOX_PARAM))
 655             elif self.expected_count >= 10000:
 656                 if details.viewbox.area < 0.5:
 657                     sql = sql.where(tsearch.c.centroid.intersects(VIEWBOX2_PARAM))
 658                 else:
 659                     sql = sql.where(tsearch.c.centroid.ST_Intersects_no_index(VIEWBOX2_PARAM))
 660             else:
 661                 penalty += sa.case((t.c.geometry.intersects(VIEWBOX_PARAM), 0.0),
 662                                    (t.c.geometry.intersects(VIEWBOX2_PARAM), 0.5),
 663                                    else_=1.0)
 664
 665         if details.near is not None:
 666             if details.near_radius is not None:
 667                 if details.near_radius < 0.1:
 668                     sql = sql.where(tsearch.c.centroid.ST_DWithin(NEAR_PARAM, NEAR_RADIUS_PARAM))
 669                 else:
 670                     sql = sql.where(tsearch.c.centroid.ST_DWithin_no_index(NEAR_PARAM,
 671                                                                            NEAR_RADIUS_PARAM))
 672             sql = sql.add_columns((-tsearch.c.centroid.ST_Distance(NEAR_PARAM))
 673                                       .label('importance'))
 674             sql = sql.order_by(sa.desc(sa.text('importance')))
 675         else:
 676             if self.expected_count < 10000\
 677                or (details.viewbox is not None and details.viewbox.area < 0.5):
 678                 sql = sql.order_by(
 679                         penalty - sa.case((tsearch.c.importance > 0, tsearch.c.importance),
 680                                     else_=0.75001-(sa.cast(tsearch.c.search_rank, sa.Float())/40)))
 681             sql = sql.add_columns(t.c.importance)
 682
 683
 684         sql = sql.add_columns(penalty.label('accuracy'))
 685
 686         if self.expected_count < 10000:
 687             sql = sql.order_by(sa.text('accuracy'))
 688
 689         if self.housenumbers:
 690             hnr_regexp = f"\\m({'|'.join(self.housenumbers.values)})\\M"
 691             sql = sql.where(tsearch.c.address_rank.between(16, 30))\
 692                      .where(sa.or_(tsearch.c.address_rank < 30,
 693                                    t.c.housenumber.op('~*')(hnr_regexp)))
 694
 695             # Cross check for housenumbers, need to do that on a rather large
 696             # set. Worst case there are 40.000 main streets in OSM.
 697             inner = sql.limit(10000).subquery()
 698
 699             # Housenumbers from placex
 700             thnr = conn.t.placex.alias('hnr')
 701             pid_list = array_agg(thnr.c.place_id) # type: ignore[no-untyped-call]
 702             place_sql = sa.select(pid_list)\
 703                           .where(thnr.c.parent_place_id == inner.c.place_id)\
 704                           .where(thnr.c.housenumber.op('~*')(hnr_regexp))\
 705                           .where(thnr.c.linked_place_id == None)\
 706                           .where(thnr.c.indexed_status == 0)
 707
 708             if details.excluded:
 709                 place_sql = place_sql.where(thnr.c.place_id.not_in(sa.bindparam('excluded')))
 710             if self.qualifiers:
 711                 place_sql = place_sql.where(self.qualifiers.sql_restrict(thnr))
 712
 713             numerals = [int(n) for n in self.housenumbers.values
 714                         if n.isdigit() and len(n) < 8]
 715             interpol_sql: SaColumn
 716             tiger_sql: SaColumn
 717             if numerals and \
 718                (not self.qualifiers or ('place', 'house') in self.qualifiers.values):
 719                 # Housenumbers from interpolations
 720                 interpol_sql = _make_interpolation_subquery(conn.t.osmline, inner,
 721                                                             numerals, details)
 722                 # Housenumbers from Tiger
 723                 tiger_sql = sa.case((inner.c.country_code == 'us',
 724                                      _make_interpolation_subquery(conn.t.tiger, inner,
 725                                                                   numerals, details)
 726                                     ), else_=None)
 727             else:
 728                 interpol_sql = sa.null()
 729                 tiger_sql = sa.null()
 730
 731             unsort = sa.select(inner, place_sql.scalar_subquery().label('placex_hnr'),
 732                                interpol_sql.label('interpol_hnr'),
 733                                tiger_sql.label('tiger_hnr')).subquery('unsort')
 734             sql = sa.select(unsort)\
 735                     .order_by(sa.case((unsort.c.placex_hnr != None, 1),
 736                                       (unsort.c.interpol_hnr != None, 2),
 737                                       (unsort.c.tiger_hnr != None, 3),
 738                                       else_=4),
 739                               unsort.c.accuracy)
 740         else:
 741             sql = sql.where(t.c.linked_place_id == None)\
 742                      .where(t.c.indexed_status == 0)
 743             if self.qualifiers:
 744                 sql = sql.where(self.qualifiers.sql_restrict(t))
 745             if details.excluded:
 746                 sql = sql.where(_exclude_places(tsearch))
 747             if details.min_rank > 0:
 748                 sql = sql.where(sa.or_(tsearch.c.address_rank >= MIN_RANK_PARAM,
 749                                        tsearch.c.search_rank >= MIN_RANK_PARAM))
 750             if details.max_rank < 30:
 751                 sql = sql.where(sa.or_(tsearch.c.address_rank <= MAX_RANK_PARAM,
 752                                        tsearch.c.search_rank <= MAX_RANK_PARAM))
 753             if details.layers is not None:
 754                 sql = sql.where(_filter_by_layer(t, details.layers))
 755
 756         sql = sql.limit(LIMIT_PARAM)
 757
 758         results = nres.SearchResults()
 759         for row in await conn.execute(sql, _details_to_bind_params(details)):
 760             result = nres.create_from_placex_row(row, nres.SearchResult)
 761             assert result
 762             result.bbox = Bbox.from_wkb(row.bbox)
 763             result.accuracy = row.accuracy
 764             if self.housenumbers and row.rank_address < 30:
 765                 if row.placex_hnr:
 766                     subs = _get_placex_housenumbers(conn, row.placex_hnr, details)
 767                 elif row.interpol_hnr:
 768                     subs = _get_osmline(conn, row.interpol_hnr, numerals, details)
 769                 elif row.tiger_hnr:
 770                     subs = _get_tiger(conn, row.tiger_hnr, numerals, row.osm_id, details)
 771                 else:
 772                     subs = None
 773
 774                 if subs is not None:
 775                     async for sub in subs:
 776                         assert sub.housenumber
 777                         sub.accuracy = result.accuracy
 778                         if not any(nr in self.housenumbers.values
 779                                    for nr in sub.housenumber.split(';')):
 780                             sub.accuracy += 0.6
 781                         results.append(sub)
 782
 783                 # Only add the street as a result, if it meets all other
 784                 # filter conditions.
 785                 if (not details.excluded or result.place_id not in details.excluded)\
 786                    and (not self.qualifiers or result.category in self.qualifiers.values)\
 787                    and result.rank_address >= details.min_rank:
 788                     result.accuracy += 1.0 # penalty for missing housenumber
 789                     results.append(result)
 790             else:
 791                 results.append(result)
 792
 793         return results