X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/371a780ef427d1b905a729da5f289b5e1099c152..e129df379493789f749d7e92b6f1f29db192d01e:/nominatim/api/v1/server_glue.py diff --git a/nominatim/api/v1/server_glue.py b/nominatim/api/v1/server_glue.py index fd4fa14b..24f2234b 100644 --- a/nominatim/api/v1/server_glue.py +++ b/nominatim/api/v1/server_glue.py @@ -15,24 +15,28 @@ import dataclasses import math from urllib.parse import urlencode +import sqlalchemy as sa + from nominatim.errors import UsageError from nominatim.config import Configuration import nominatim.api as napi import nominatim.api.logging as loglib from nominatim.api.v1.format import dispatch as formatting +from nominatim.api.v1.format import RawDataList from nominatim.api.v1 import helpers -CONTENT_TYPE = { - 'text': 'text/plain; charset=utf-8', - 'xml': 'text/xml; charset=utf-8', - 'debug': 'text/html; charset=utf-8' -} +CONTENT_TEXT = 'text/plain; charset=utf-8' +CONTENT_XML = 'text/xml; charset=utf-8' +CONTENT_HTML = 'text/html; charset=utf-8' +CONTENT_JSON = 'application/json; charset=utf-8' + +CONTENT_TYPE = {'text': CONTENT_TEXT, 'xml': CONTENT_XML, 'debug': CONTENT_HTML} class ASGIAdaptor(abc.ABC): """ Adapter class for the different ASGI frameworks. Wraps functionality over concrete requests and responses. """ - content_type: str = 'text/plain; charset=utf-8' + content_type: str = CONTENT_TEXT @abc.abstractmethod def get(self, name: str, default: Optional[str] = None) -> Optional[str]: @@ -55,7 +59,7 @@ class ASGIAdaptor(abc.ABC): @abc.abstractmethod - def create_response(self, status: int, output: str) -> Any: + def create_response(self, status: int, output: str, num_results: int) -> Any: """ Create a response from the given parameters. The result will be returned by the endpoint functions. The adaptor may also return None when the response is created internally with some @@ -66,6 +70,11 @@ class ASGIAdaptor(abc.ABC): body of the response to 'output'. """ + @abc.abstractmethod + def base_uri(self) -> str: + """ Return the URI of the original request. + """ + @abc.abstractmethod def config(self) -> Configuration: @@ -73,19 +82,19 @@ class ASGIAdaptor(abc.ABC): """ - def build_response(self, output: str, status: int = 200) -> Any: + def build_response(self, output: str, status: int = 200, num_results: int = 0) -> Any: """ Create a response from the given output. Wraps a JSONP function around the response, if necessary. """ - if self.content_type == 'application/json' and status == 200: + if self.content_type == CONTENT_JSON and status == 200: jsonp = self.get('json_callback') if jsonp is not None: if any(not part.isidentifier() for part in jsonp.split('.')): self.raise_error('Invalid json_callback value') output = f"{jsonp}({output})" - self.content_type = 'application/javascript' + self.content_type = 'application/javascript; charset=utf-8' - return self.create_response(status, output) + return self.create_response(status, output, num_results) def raise_error(self, msg: str, status: int = 400) -> NoReturn: @@ -93,16 +102,16 @@ class ASGIAdaptor(abc.ABC): message. The message will be formatted according to the output format chosen by the request. """ - if self.content_type == 'text/xml; charset=utf-8': + if self.content_type == CONTENT_XML: msg = f""" {status} {msg} """ - elif self.content_type == 'application/json': + elif self.content_type == CONTENT_JSON: msg = f"""{{"error":{{"code":{status},"message":"{msg}"}}}}""" - elif self.content_type == 'text/html; charset=utf-8': + elif self.content_type == CONTENT_HTML: loglib.log().section('Execution error') loglib.log().var_dump('Status', status) loglib.log().var_dump('Message', msg) @@ -185,7 +194,7 @@ class ASGIAdaptor(abc.ABC): """ Return the accepted languages. """ return self.get('accept-language')\ - or self.get_header('http_accept_language')\ + or self.get_header('accept-language')\ or self.config().DEFAULT_LANGUAGE @@ -196,7 +205,7 @@ class ASGIAdaptor(abc.ABC): """ if self.get_bool('debug', False): loglib.set_log_output('html') - self.content_type = 'text/html; charset=utf-8' + self.content_type = CONTENT_HTML return True return False @@ -226,7 +235,7 @@ class ASGIAdaptor(abc.ABC): self.raise_error("Parameter 'format' must be one of: " + ', '.join(formatting.list_formats(result_type))) - self.content_type = CONTENT_TYPE.get(fmt, 'application/json') + self.content_type = CONTENT_TYPE.get(fmt, CONTENT_JSON) return fmt @@ -250,7 +259,7 @@ class ASGIAdaptor(abc.ABC): numgeoms += 1 if numgeoms > self.config().get_int('POLYGON_OUTPUT_MAX_TYPES'): - self.raise_error('Too many polgyon output options selected.') + self.raise_error('Too many polygon output options selected.') return {'address_details': True, 'geometry_simplification': self.get_float('polygon_threshold', 0.0), @@ -294,7 +303,7 @@ async def details_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> result = await api.details(place, address_details=params.get_bool('addressdetails', False), - linked_places=params.get_bool('linkedplaces', False), + linked_places=params.get_bool('linkedplaces', True), parented_places=params.get_bool('hierarchy', False), keywords=params.get_bool('keywords', False), geometry_output = napi.GeometryFormat.GEOJSON @@ -315,7 +324,7 @@ async def details_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> 'group_hierarchy': params.get_bool('group_hierarchy', False), 'icon_base_url': params.config().MAPICON_URL}) - return params.build_response(output) + return params.build_response(output, num_results=1) async def reverse_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any: @@ -332,7 +341,7 @@ async def reverse_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> result = await api.reverse(coord, **details) if debug: - return params.build_response(loglib.get_and_disable()) + return params.build_response(loglib.get_and_disable(), num_results=1 if result else 0) if fmt == 'xml': queryparts = {'lat': str(coord.lat), 'lon': str(coord.lon), 'format': 'xml'} @@ -354,7 +363,7 @@ async def reverse_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> output = formatting.format_result(napi.ReverseResults([result] if result else []), fmt, fmt_options) - return params.build_response(output) + return params.build_response(output, num_results=1 if result else 0) async def lookup_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any: @@ -370,13 +379,16 @@ async def lookup_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A if len(oid) > 1 and oid[0] in 'RNWrnw' and oid[1:].isdigit(): places.append(napi.OsmID(oid[0], int(oid[1:]))) + if len(places) > params.config().get_int('LOOKUP_MAX_COUNT'): + params.raise_error('Too many object IDs.') + if places: results = await api.lookup(places, **details) else: results = napi.SearchResults() if debug: - return params.build_response(loglib.get_and_disable()) + return params.build_response(loglib.get_and_disable(), num_results=len(results)) fmt_options = {'extratags': params.get_bool('extratags', False), 'namedetails': params.get_bool('namedetails', False), @@ -386,7 +398,7 @@ async def lookup_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A output = formatting.format_result(results, fmt, fmt_options) - return params.build_response(output) + return params.build_response(output, num_results=len(results)) async def _unstructured_search(query: str, api: napi.NominatimAPIAsync, @@ -439,18 +451,29 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A details['min_rank'], details['max_rank'] = \ helpers.feature_type_to_rank(params.get('featureType', '')) + if params.get('featureType', None) is not None: + details['layers'] = napi.DataLayer.ADDRESS + else: + details['layers'] = params.get_layers() + # unstructured query parameters query = params.get('q', None) + # structured query parameters queryparts = {} + for key in ('amenity', 'street', 'city', 'county', 'state', 'postalcode', 'country'): + details[key] = params.get(key, None) + if details[key]: + queryparts[key] = details[key] + try: if query is not None: + if queryparts: + params.raise_error("Structured query parameters" + "(amenity, street, city, county, state, postalcode, country)" + " cannot be used together with 'q' parameter.") queryparts['q'] = query results = await _unstructured_search(query, api, details) else: - for key in ('amenity', 'street', 'city', 'county', 'state', 'postalcode', 'country'): - details[key] = params.get(key, None) - if details[key]: - queryparts[key] = details[key] query = ', '.join(queryparts.values()) results = await api.search_address(**details) @@ -463,7 +486,7 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A results = helpers.deduplicate_results(results, max_results) if debug: - return params.build_response(loglib.get_and_disable()) + return params.build_response(loglib.get_and_disable(), num_results=len(results)) if fmt == 'xml': helpers.extend_query_parts(queryparts, details, @@ -473,7 +496,7 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A (str(r.place_id) for r in results if r.place_id)) queryparts['format'] = fmt - moreurl = urlencode(queryparts) + moreurl = params.base_uri() + '/search?' + urlencode(queryparts) else: moreurl = '' @@ -486,7 +509,59 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A output = formatting.format_result(results, fmt, fmt_options) - return params.build_response(output) + return params.build_response(output, num_results=len(results)) + + +async def deletable_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any: + """ Server glue for /deletable endpoint. + This is a special endpoint that shows polygons that have been + deleted or are broken in the OSM data but are kept in the + Nominatim database to minimize disruption. + """ + fmt = params.parse_format(RawDataList, 'json') + + async with api.begin() as conn: + sql = sa.text(""" SELECT p.place_id, country_code, + name->'name' as name, i.* + FROM placex p, import_polygon_delete i + WHERE p.osm_id = i.osm_id AND p.osm_type = i.osm_type + AND p.class = i.class AND p.type = i.type + """) + results = RawDataList(r._asdict() for r in await conn.execute(sql)) + + return params.build_response(formatting.format_result(results, fmt, {})) + + +async def polygons_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any: + """ Server glue for /polygons endpoint. + This is a special endpoint that shows polygons that have changed + thier size but are kept in the Nominatim database with their + old area to minimize disruption. + """ + fmt = params.parse_format(RawDataList, 'json') + sql_params: Dict[str, Any] = { + 'days': params.get_int('days', -1), + 'cls': params.get('class') + } + reduced = params.get_bool('reduced', False) + + async with api.begin() as conn: + sql = sa.select(sa.text("""osm_type, osm_id, class, type, + name->'name' as name, + country_code, errormessage, updated"""))\ + .select_from(sa.text('import_polygon_error')) + if sql_params['days'] > 0: + sql = sql.where(sa.text("updated > 'now'::timestamp - make_interval(days => :days)")) + if reduced: + sql = sql.where(sa.text("errormessage like 'Area reduced%'")) + if sql_params['cls'] is not None: + sql = sql.where(sa.text("class = :cls")) + + sql = sql.order_by(sa.literal_column('updated').desc()).limit(1000) + + results = RawDataList(r._asdict() for r in await conn.execute(sql, sql_params)) + + return params.build_response(formatting.format_result(results, fmt, {})) EndpointFunc = Callable[[napi.NominatimAPIAsync, ASGIAdaptor], Any] @@ -496,5 +571,7 @@ ROUTES = [ ('details', details_endpoint), ('reverse', reverse_endpoint), ('lookup', lookup_endpoint), - ('search', search_endpoint) + ('search', search_endpoint), + ('deletable', deletable_endpoint), + ('polygons', polygons_endpoint), ]