shell: bash
- name: Install${{ matrix.flavour }} prerequisites
run: |
- sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
+ sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson
if [ "$FLAVOUR" == "oldstuff" ]; then
pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg
else
if: matrix.flavour == 'oldstuff'
- name: Install Python webservers
- run: pip3 install falcon sanic sanic-testing sanic-cors starlette
+ run: pip3 install falcon starlette
- name: Install latest pylint
run: pip3 install -U pylint asgi_lifespan
- name: Prepare import environment
run: |
mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
+ mv Nominatim/settings/flex-base.lua flex-base.lua
+ mv Nominatim/settings/import-extratags.lua import-extratags.lua
+ mv Nominatim/settings/taginfo.lua taginfo.lua
rm -rf Nominatim
mkdir data-env-reverse
working-directory: /home/nominatim
run: nominatim --version
working-directory: /home/nominatim/nominatim-project
+ - name: Print taginfo
+ run: lua taginfo.lua
+ working-directory: /home/nominatim
+
- name: Collect host OS information
run: nominatim admin --collect-os-info
working-directory: /home/nominatim/nominatim-project
* one of the following web frameworks:
* [falcon](https://falconframework.org/) (3.0+)
- * [sanic](https://sanic.dev) and (optionally) [sanic-cors](https://github.com/ashleysommer/sanic-cors)
* [starlette](https://www.starlette.io/)
- * [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks)
+ * [uvicorn](https://www.uvicorn.org/)
For dependencies for running tests and building documentation, see
the [Development section](../develop/Development-Environment.md).
For testing the Python search frontend, you need to install extra dependencies
depending on your choice of webserver framework:
-* [sanic-testing](https://sanic.dev/en/plugins/sanic-testing/getting-started.html) (sanic only)
* [httpx](https://www.python-httpx.org/) (starlette only)
* [asgi-lifespan](https://github.com/florimondmanca/asgi-lifespan) (starlette only)
pip3 install --user behave mkdocs mkdocstrings pytest pytest-asyncio pylint \
mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \
types-ujson types-requests types-Pygments typing-extensions\
- sanic-testing httpx asgi-lifespan
+ httpx asgi-lifespan
```
The `mkdocs` executable will be located in `.local/bin`. You may have to add
self._write(f"rank={res.rank_address}, ")
self._write(f"osm={format_osm(res.osm_object)}, ")
self._write(f'cc={res.country_code}, ')
- self._write(f'importance={res.importance or -1:.5f})</dd>')
+ self._write(f'importance={res.importance or float("nan"):.5f})</dd>')
total += 1
self._write(f'</dl><b>TOTAL:</b> {total}</p>')
def _python_var(self, var: Any) -> str:
if CODE_HIGHLIGHT:
- fmt = highlight(repr(var), PythonLexer(), HtmlFormatter(nowrap=True))
+ fmt = highlight(str(var), PythonLexer(), HtmlFormatter(nowrap=True))
return f'<div class="highlight"><code class="lang-python">{fmt}</code></div>'
return f'<code class="lang-python">{str(var)}</code>'
yield dbs.CountrySearch(sdata)
if sdata.postcodes and (is_category or self.configured_for_postcode):
+ penalty = 0.0 if sdata.countries else 0.1
if address:
sdata.lookups = [dbf.FieldLookup('nameaddress_vector',
[t.token for r in address
for t in self.query.get_partials_list(r)],
'restrict')]
- yield dbs.PostcodeSearch(0.4, sdata)
+ penalty += 0.2
+ yield dbs.PostcodeSearch(penalty, sdata)
def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],
details: SearchDetails) -> nres.SearchResults:
""" Look up the country in the fallback country tables.
"""
+ # Avoid the fallback search when this is a more search. Country results
+ # usually are in the first batch of results and it is not possible
+ # to exclude these fallbacks.
+ if details.excluded:
+ return nres.SearchResults()
+
t = conn.t.country_name
tgrid = conn.t.country_grid
sql = sql.where(tsearch.c.country_code.in_(self.countries.values))
if self.postcodes:
+ # if a postcode is given, don't search for state or country level objects
+ sql = sql.where(tsearch.c.address_rank > 9)
tpc = conn.t.postcode
if self.expected_count > 1000:
# Many results expected. Restrict by postcode.
return f'{c[0]}^{c[1]}'
for search in searches[start:]:
- fields = ('name_lookups', 'name_ranking', 'countries', 'housenumbers',
+ fields = ('lookups', 'rankings', 'countries', 'housenumbers',
'postcodes', 'qualifier')
iters = itertools.zip_longest([f"{search.penalty:.3g}"],
*(getattr(search, attr, []) for attr in fields),
"""
log().section('Analyze query (using ICU tokenizer)')
normalized = list(filter(lambda p: p.text,
- (qmod.Phrase(p.ptype, self.normalizer.transliterate(p.text))
+ (qmod.Phrase(p.ptype, self.normalize_text(p.text))
for p in phrases)))
query = qmod.QueryStruct(normalized)
log().var_dump('Normalized query', query.source)
return query
+ def normalize_text(self, text: str) -> str:
+ """ Bring the given text into a normalized form. That is the
+ standardized form search will work with. All information removed
+ at this stage is inevitably lost.
+ """
+ return cast(str, self.normalizer.transliterate(text))
+
+
def split_query(self, query: qmod.QueryStruct) -> Tuple[QueryParts, WordDict]:
""" Transliterate the phrases and split them into tokens.
and (repl.ttype != qmod.TokenType.HOUSENUMBER
or len(tlist.tokens[0].lookup_word) > 4):
repl.add_penalty(0.39)
- elif tlist.ttype == qmod.TokenType.HOUSENUMBER:
+ elif tlist.ttype == qmod.TokenType.HOUSENUMBER \
+ and len(tlist.tokens[0].lookup_word) <= 3:
if any(c.isdigit() for c in tlist.tokens[0].lookup_word):
for repl in node.starting:
- if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER \
- and (repl.ttype != qmod.TokenType.HOUSENUMBER
- or len(tlist.tokens[0].lookup_word) <= 3):
+ if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER:
repl.add_penalty(0.5 - tlist.tokens[0].penalty)
elif tlist.ttype not in (qmod.TokenType.COUNTRY, qmod.TokenType.PARTIAL):
norm = parts[i].normalized
and (repl.ttype != qmod.TokenType.HOUSENUMBER
or len(tlist.tokens[0].lookup_word) > 4):
repl.add_penalty(0.39)
- elif tlist.ttype == qmod.TokenType.HOUSENUMBER:
+ elif tlist.ttype == qmod.TokenType.HOUSENUMBER \
+ and len(tlist.tokens[0].lookup_word) <= 3:
if any(c.isdigit() for c in tlist.tokens[0].lookup_word):
for repl in node.starting:
- if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER \
- and (repl.ttype != qmod.TokenType.HOUSENUMBER
- or len(tlist.tokens[0].lookup_word) <= 3):
+ if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER:
repl.add_penalty(0.5 - tlist.tokens[0].penalty)
if (base.postcode.start == 0 and self.direction != -1)\
or (base.postcode.end == query.num_token_slots() and self.direction != 1):
log().comment('postcode search')
- yield dataclasses.replace(base, penalty=self.penalty)
+ # <address>,<postcode> should give preference to address search
+ if base.postcode.start == 0:
+ penalty = self.penalty
+ else:
+ penalty = self.penalty + 0.1
+ yield dataclasses.replace(base, penalty=penalty)
# Postcode or country-only search
if not base.address:
log().comment('postcode/country search')
yield dataclasses.replace(base, penalty=self.penalty)
else:
+ # <postcode>,<address> should give preference to postcode search
+ if base.postcode and base.postcode.start == 0:
+ self.penalty += 0.1
# Use entire first word as name
if self.direction != -1:
log().comment('first word = name')
else:
raise UsageError("Parameter 'excluded' needs to be a comma-separated list "
"or a Python list of numbers.")
- if not all(isinstance(i, int) or (isinstance(i, str) and i.isdigit()) for i in plist):
+ if not all(isinstance(i, int) or
+ (isinstance(i, str) and (not i or i.isdigit())) for i in plist):
raise UsageError("Parameter 'excluded' only takes place IDs.")
- return [int(id) for id in plist if id]
+ return [int(id) for id in plist if id] or [0]
def format_categories(categories: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
"""
parsed = SearchDetails.from_kwargs(details)
if parsed.geometry_output != GeometryFormat.NONE:
- if parsed.geometry_output & GeometryFormat.GEOJSON:
+ if GeometryFormat.GEOJSON in parsed.geometry_output:
queryparts['polygon_geojson'] = '1'
- if parsed.geometry_output & GeometryFormat.KML:
+ if GeometryFormat.KML in parsed.geometry_output:
queryparts['polygon_kml'] = '1'
- if parsed.geometry_output & GeometryFormat.SVG:
+ if GeometryFormat.SVG in parsed.geometry_output:
queryparts['polygon_svg'] = '1'
- if parsed.geometry_output & GeometryFormat.TEXT:
+ if GeometryFormat.TEXT in parsed.geometry_output:
queryparts['polygon_text'] = '1'
if parsed.address_details:
queryparts['addressdetails'] = '1'
""" Return the accepted languages.
"""
return self.get('accept-language')\
- or self.get_header('http_accept_language')\
+ or self.get_header('accept-language')\
or self.config().DEFAULT_LANGUAGE
group.add_argument('--server', default='127.0.0.1:8088',
help='The address the server will listen to.')
group.add_argument('--engine', default='php',
- choices=('php', 'sanic', 'falcon', 'starlette'),
+ choices=('php', 'falcon', 'starlette'),
help='Webserver framework to run. (default: php)')
if args.engine == 'php':
run_php_server(args.server, args.project_dir / 'website')
else:
+ import uvicorn # pylint: disable=import-outside-toplevel
server_info = args.server.split(':', 1)
host = server_info[0]
if len(server_info) > 1:
else:
port = 8088
- if args.engine == 'sanic':
- server_module = importlib.import_module('nominatim.server.sanic.server')
+ server_module = importlib.import_module(f'nominatim.server.{args.engine}.server')
- app = server_module.get_application(args.project_dir)
- app.run(host=host, port=port, debug=True, single_process=True)
- else:
- import uvicorn # pylint: disable=import-outside-toplevel
-
- if args.engine == 'falcon':
- server_module = importlib.import_module('nominatim.server.falcon.server')
- elif args.engine == 'starlette':
- server_module = importlib.import_module('nominatim.server.starlette.server')
-
- app = server_module.get_application(args.project_dir)
- uvicorn.run(app, host=host, port=port)
+ app = server_module.get_application(args.project_dir)
+ uvicorn.run(app, host=host, port=port)
return 0
+++ /dev/null
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# This file is part of Nominatim. (https://nominatim.org)
-#
-# Copyright (C) 2023 by the Nominatim developer community.
-# For a full list of authors see the git log.
-"""
-Server implementation using the sanic webserver framework.
-"""
-from typing import Any, Optional, Mapping, Callable, cast, Coroutine
-from pathlib import Path
-
-from sanic import Request, HTTPResponse, Sanic
-from sanic.exceptions import SanicException
-from sanic.response import text as TextResponse
-
-from nominatim.api import NominatimAPIAsync
-import nominatim.api.v1 as api_impl
-from nominatim.config import Configuration
-
-class ParamWrapper(api_impl.ASGIAdaptor):
- """ Adaptor class for server glue to Sanic framework.
- """
-
- def __init__(self, request: Request) -> None:
- self.request = request
-
-
- def get(self, name: str, default: Optional[str] = None) -> Optional[str]:
- return cast(Optional[str], self.request.args.get(name, default))
-
-
- def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
- return cast(Optional[str], self.request.headers.get(name, default))
-
-
- def error(self, msg: str, status: int = 400) -> SanicException:
- exception = SanicException(msg, status_code=status)
-
- return exception
-
-
- def create_response(self, status: int, output: str) -> HTTPResponse:
- return TextResponse(output, status=status, content_type=self.content_type)
-
-
- def config(self) -> Configuration:
- return cast(Configuration, self.request.app.ctx.api.config)
-
-
-def _wrap_endpoint(func: api_impl.EndpointFunc)\
- -> Callable[[Request], Coroutine[Any, Any, HTTPResponse]]:
- async def _callback(request: Request) -> HTTPResponse:
- return cast(HTTPResponse, await func(request.app.ctx.api, ParamWrapper(request)))
-
- return _callback
-
-
-def get_application(project_dir: Path,
- environ: Optional[Mapping[str, str]] = None) -> Sanic:
- """ Create a Nominatim sanic ASGI application.
- """
- app = Sanic("NominatimInstance")
-
- app.ctx.api = NominatimAPIAsync(project_dir, environ)
-
- if app.ctx.api.config.get_bool('CORS_NOACCESSCONTROL'):
- from sanic_cors import CORS # pylint: disable=import-outside-toplevel
- CORS(app)
-
- legacy_urls = app.ctx.api.config.get_bool('SERVE_LEGACY_URLS')
- for name, func in api_impl.ROUTES:
- endpoint = _wrap_endpoint(func)
- app.add_route(endpoint, f"/{name}", name=f"v1_{name}_simple")
- if legacy_urls:
- app.add_route(endpoint, f"/{name}.php", name=f"v1_{name}_legacy")
-
- return app
local SAVE_EXTRA_MAINS = false
local POSTCODE_FALLBACK = true
+-- tables required for taginfo
+module.TAGINFO_MAIN = {keys = {}, delete_tags = {}}
+module.TAGINFO_NAME_KEYS = {}
+module.TAGINFO_ADDRESS_KEYS = {}
+
-- The single place table.
local place_table = osm2pgsql.define_table{
end
end
+-- Returns prefix part of the keys, and reject suffix matching keys
+local function process_key(key)
+ if key:sub(1, 1) == '*' then
+ return nil
+ end
+ if key:sub(#key, #key) == '*' then
+ return key:sub(1, #key - 2)
+ end
+ return key
+end
+
-- Process functions for all data types
function module.process_node(object)
PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
PRE_EXTRAS = module.tag_match{keys = data.extra_keys,
tags = data.extra_tags}
+ module.TAGINFO_MAIN.delete_tags = data.delete_tags
end
function module.set_main_tags(data)
MAIN_KEYS = data
+ local keys = {}
+ for k, _ in pairs(data) do
+ table.insert(keys, k)
+ end
+ module.TAGINFO_MAIN.keys = keys
end
function module.set_name_tags(data)
NAMES = module.tag_group(data)
+
+ for _, lst in pairs(data) do
+ for _, k in ipairs(lst) do
+ local key = process_key(k)
+ if key ~= nil then
+ module.TAGINFO_NAME_KEYS[key] = true
+ end
+ end
+ end
end
function module.set_address_tags(data)
POSTCODE_FALLBACK = data.postcode_fallback
data.postcode_fallback = nil
end
-
ADDRESS_TAGS = module.tag_group(data)
+
+ for _, lst in pairs(data) do
+ if lst ~= nil then
+ for _, k in ipairs(lst) do
+ local key = process_key(k)
+ if key ~= nil then
+ module.TAGINFO_ADDRESS_KEYS[key] = true
+ end
+ end
+ end
+ end
end
function module.set_unused_handling(data)
historic = 'always',
military = 'always',
natural = 'named',
- landuse = 'named',
highway = {'always',
street_lamp = 'named',
traffic_signals = 'named',
historic = 'always',
military = 'always',
natural = 'named',
- landuse = 'named',
highway = {'always',
street_lamp = 'named',
traffic_signals = 'named',
--- /dev/null
+-- Prints taginfo project description in the standard output
+--
+
+-- create fake "osm2pgsql" table for flex-base, originally created by the main C++ program
+osm2pgsql = {}
+function osm2pgsql.define_table(...) end
+
+-- provide path to flex-style lua file
+flex = require('import-extratags')
+local json = require ('dkjson')
+
+
+------------ helper functions ---------------------
+
+function get_key_description(key, description)
+ local desc = {}
+ desc.key = key
+ desc.description = description
+ set_keyorder(desc, {'key', 'description'})
+ return desc
+end
+
+-- Sets the key order for the resulting JSON table
+function set_keyorder(table, order)
+ setmetatable(table, {
+ __jsonorder = order
+ })
+end
+
+
+-- Prints the collected tags in the required format in JSON
+function print_taginfo()
+ local tags = {}
+
+ for _, k in ipairs(flex.TAGINFO_MAIN.keys) do
+ local desc = get_key_description(k, 'POI/feature in the search database')
+ if flex.TAGINFO_MAIN.delete_tags[k] ~= nil then
+ desc.description = string.format('%s(except for values: %s).', desc.description,
+ table.concat(flex.TAGINFO_MAIN.delete_tags[k], ', '))
+ end
+ table.insert(tags, desc)
+ end
+
+ for k, _ in pairs(flex.TAGINFO_NAME_KEYS) do
+ local desc = get_key_description(k, 'Searchable name of the place.')
+ table.insert(tags, desc)
+ end
+ for k, _ in pairs(flex.TAGINFO_ADDRESS_KEYS) do
+ local desc = get_key_description(k, 'Used to determine the address of a place.')
+ table.insert(tags, desc)
+ end
+
+ local format = {
+ data_format = 1,
+ data_url = 'https://nominatim.openstreetmap.org/taginfo.json',
+ project = {
+ name = 'Nominatim',
+ description = 'OSM search engine.',
+ project_url = 'https://nominatim.openstreetmap.org',
+ doc_url = 'https://nominatim.org/release-docs/develop/',
+ contact_name = 'Sarah Hoffmann',
+ contact_email = 'lonvia@denofr.de'
+ }
+ }
+ format.tags = tags
+
+ set_keyorder(format, {'data_format', 'data_url', 'project', 'tags'})
+ set_keyorder(format.project, {'name', 'description', 'project_url', 'doc_url',
+ 'contact_name', 'contact_email'})
+
+ print(json.encode(format))
+end
+
+print_taginfo()
return _request
- def create_api_request_func_sanic(self):
- import nominatim.server.sanic.server
-
- async def _request(endpoint, params, project_dir, environ, http_headers):
- app = nominatim.server.sanic.server.get_application(project_dir, environ)
-
- _, response = await app.asgi_client.get(f"/{endpoint}", params=params,
- headers=http_headers)
-
- return response.text, response.status_code
-
- return _request
-
-
def create_api_request_func_falcon(self):
import nominatim.server.falcon.server
import falcon.testing
assert query.num_token_slots() == 1
- torder = [(tl.tokens[0].penalty, tl.ttype) for tl in query.nodes[0].starting]
+ torder = [(tl.tokens[0].penalty, tl.ttype.name) for tl in query.nodes[0].starting]
torder.sort()
- assert [t[1] for t in torder] == [TokenType[o] for o in order]
+ assert [t[1] for t in torder] == order
@pytest.mark.asyncio
async def test_category_words_only_at_beginning(conn):
assert query.num_token_slots() == 1
- torder = [(tl.tokens[0].penalty, tl.ttype) for tl in query.nodes[0].starting]
- print(query.nodes[0].starting)
+ torder = [(tl.tokens[0].penalty, tl.ttype.name) for tl in query.nodes[0].starting]
torder.sort()
- assert [t[1] for t in torder] == [TokenType[o] for o in order]
+ assert [t[1] for t in torder] == order
@pytest.mark.asyncio
(BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
check_assignments(yield_token_assignments(q),
- TokenAssignment(name=TokenRange(1, 2),
+ TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
postcode=TokenRange(0, 1)),
TokenAssignment(postcode=TokenRange(0, 1),
address=[TokenRange(1, 2)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(name=TokenRange(0, 1),
postcode=TokenRange(1, 2)),
- TokenAssignment(postcode=TokenRange(1, 2),
+ TokenAssignment(penalty=0.1, postcode=TokenRange(1, 2),
address=[TokenRange(0, 1)]))
def test_accepted_languages_from_header():
- a = FakeAdaptor(headers={'http_accept_language': 'de'})
+ a = FakeAdaptor(headers={'accept-language': 'de'})
assert a.get_accepted_languages() == 'de'
def test_accepted_languages_param_over_header():
a = FakeAdaptor(params={'accept-language': 'de'},
- headers={'http_accept_language': 'en'})
+ headers={'accept-language': 'en'})
assert a.get_accepted_languages() == 'de'
def test_accepted_languages_header_over_default(monkeypatch):
monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'en')
- a = FakeAdaptor(headers={'http_accept_language': 'de'})
+ a = FakeAdaptor(headers={'accept-language': 'de'})
assert a.get_accepted_languages() == 'de'
loglib.log().section('Ongoing')
with pytest.raises(FakeError) as excinfo:
- a.raise_error('bad state')
+ a.raise_error('badstate')
content = ET.fromstring(excinfo.value.msg)
assert content.tag == 'html'
assert '>Ongoing<' in excinfo.value.msg
- assert 'bad state' in excinfo.value.msg
+ assert 'badstate' in excinfo.value.msg
# ASGIAdaptor.build_response
assert func.called == 1
-def test_cli_serve_sanic(cli_call, mock_func_factory):
- mod = pytest.importorskip("sanic")
- func = mock_func_factory(mod.Sanic, "run")
-
- cli_call('serve', '--engine', 'sanic') == 0
-
- assert func.called == 1
-
-
def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory):
pytest.importorskip("starlette")
mod = pytest.importorskip("uvicorn")
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
- libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
+ libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \
postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
- libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
+ libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \
postgresql-server-dev-14 postgresql-14-postgis-3 \
postgresql-contrib-14 postgresql-14-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
projects / nominatim.git / commitdiff