From: Sarah Hoffmann Date: Sun, 9 Mar 2025 16:34:04 +0000 (+0100) Subject: enable flake for bdd test code X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/78f839fbd3ccc623b64895059b440e355c06c7c3?ds=sidebyside enable flake for bdd test code --- diff --git a/.flake8 b/.flake8 index f3e0db56..cf87715a 100644 --- a/.flake8 +++ b/.flake8 @@ -8,3 +8,4 @@ per-file-ignores = __init__.py: F401 test/python/utils/test_json_writer.py: E131 test/python/conftest.py: E402 + test/bdd/*: F821 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 20da6e0b..a8bf957f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -100,7 +100,7 @@ jobs: run: ./venv/bin/pip install -U flake8 - name: Python linting - run: ../venv/bin/python -m flake8 src test/python + run: ../venv/bin/python -m flake8 src test/python test/bdd working-directory: Nominatim - name: Install mypy and typechecking info diff --git a/Makefile b/Makefile index 72072a59..f35c9782 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ pytest: pytest test/python lint: - flake8 src test/python + flake8 src test/python test/bdd bdd: cd test/bdd; behave -DREMOVE_TEMPLATE=1 diff --git a/test/bdd/environment.py b/test/bdd/environment.py index 7535c508..bedbe8d2 100644 --- a/test/bdd/environment.py +++ b/test/bdd/environment.py @@ -2,43 +2,45 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. from pathlib import Path import sys -from behave import * +from behave import * # noqa sys.path.insert(1, str(Path(__file__, '..', '..', '..', 'src').resolve())) -from steps.geometry_factory import GeometryFactory -from steps.nominatim_environment import NominatimEnvironment +from steps.geometry_factory import GeometryFactory # noqa: E402 +from steps.nominatim_environment import NominatimEnvironment # noqa: E402 TEST_BASE_DIR = Path(__file__, '..', '..').resolve() userconfig = { - 'REMOVE_TEMPLATE' : False, - 'KEEP_TEST_DB' : False, - 'DB_HOST' : None, - 'DB_PORT' : None, - 'DB_USER' : None, - 'DB_PASS' : None, - 'TEMPLATE_DB' : 'test_template_nominatim', - 'TEST_DB' : 'test_nominatim', - 'API_TEST_DB' : 'test_api_nominatim', - 'API_TEST_FILE' : TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf', - 'TOKENIZER' : None, # Test with a custom tokenizer - 'STYLE' : 'extratags', + 'REMOVE_TEMPLATE': False, + 'KEEP_TEST_DB': False, + 'DB_HOST': None, + 'DB_PORT': None, + 'DB_USER': None, + 'DB_PASS': None, + 'TEMPLATE_DB': 'test_template_nominatim', + 'TEST_DB': 'test_nominatim', + 'API_TEST_DB': 'test_api_nominatim', + 'API_TEST_FILE': TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf', + 'TOKENIZER': None, # Test with a custom tokenizer + 'STYLE': 'extratags', 'API_ENGINE': 'falcon' } -use_step_matcher("re") + +use_step_matcher("re") # noqa: F405 + def before_all(context): # logging setup context.config.setup_logging() # set up -D options - for k,v in userconfig.items(): + for k, v in userconfig.items(): context.config.userdata.setdefault(k, v) # Nominatim test setup context.nominatim = NominatimEnvironment(context.config.userdata) @@ -46,7 +48,7 @@ def before_all(context): def before_scenario(context, scenario): - if not 'SQLITE' in context.tags \ + if 'SQLITE' not in context.tags \ and context.config.userdata['API_TEST_DB'].startswith('sqlite:'): context.scenario.skip("Not usable with Sqlite database.") elif 'DB' in context.tags: @@ -56,6 +58,7 @@ def before_scenario(context, scenario): elif 'UNKNOWNDB' in context.tags: context.nominatim.setup_unknown_db() + def after_scenario(context, scenario): if 'DB' in context.tags: context.nominatim.teardown_db(context) diff --git a/test/bdd/steps/check_functions.py b/test/bdd/steps/check_functions.py index 708de852..df9e6f35 100644 --- a/test/bdd/steps/check_functions.py +++ b/test/bdd/steps/check_functions.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2023 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Collection of assertion functions used for the steps. @@ -11,9 +11,10 @@ import json import math import re -OSM_TYPE = {'N' : 'node', 'W' : 'way', 'R' : 'relation', - 'n' : 'node', 'w' : 'way', 'r' : 'relation', - 'node' : 'n', 'way' : 'w', 'relation' : 'r'} + +OSM_TYPE = {'N': 'node', 'W': 'way', 'R': 'relation', + 'n': 'node', 'w': 'way', 'r': 'relation', + 'node': 'n', 'way': 'w', 'relation': 'r'} class OsmType: @@ -23,11 +24,9 @@ class OsmType: def __init__(self, value): self.value = value - def __eq__(self, other): return other == self.value or other == OSM_TYPE[self.value] - def __str__(self): return f"{self.value} or {OSM_TYPE[self.value]}" @@ -81,7 +80,6 @@ class Bbox: return str(self.coord) - def check_for_attributes(obj, attrs, presence='present'): """ Check that the object has the given attributes. 'attrs' is a string with a comma-separated list of attributes. If 'presence' @@ -99,4 +97,3 @@ def check_for_attributes(obj, attrs, presence='present'): else: assert attr in obj, \ f"No attribute '{attr}'. Full response:\n{_dump_json()}" - diff --git a/test/bdd/steps/geometry_alias.py b/test/bdd/steps/geometry_alias.py index a9b4ec8c..dbec5201 100644 --- a/test/bdd/steps/geometry_alias.py +++ b/test/bdd/steps/geometry_alias.py @@ -2,261 +2,261 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Collection of aliases for various world coordinates. """ ALIASES = { -# Country aliases -'AD': (1.58972, 42.54241), -'AE': (54.61589, 24.82431), -'AF': (65.90264, 34.84708), -'AG': (-61.72430, 17.069), -'AI': (-63.10571, 18.25461), -'AL': (19.84941, 40.21232), -'AM': (44.64229, 40.37821), -'AO': (16.21924, -12.77014), -'AQ': (44.99999, -75.65695), -'AR': (-61.10759, -34.37615), -'AS': (-170.68470, -14.29307), -'AT': (14.25747, 47.36542), -'AU': (138.23155, -23.72068), -'AW': (-69.98255, 12.555), -'AX': (19.91839, 59.81682), -'AZ': (48.38555, 40.61639), -'BA': (17.18514, 44.25582), -'BB': (-59.53342, 13.19), -'BD': (89.75989, 24.34205), -'BE': (4.90078, 50.34682), -'BF': (-0.56743, 11.90471), -'BG': (24.80616, 43.09859), -'BH': (50.52032, 25.94685), -'BI': (29.54561, -2.99057), -'BJ': (2.70062, 10.02792), -'BL': (-62.79349, 17.907), -'BM': (-64.77406, 32.30199), -'BN': (114.52196, 4.28638), -'BO': (-62.02473, -17.77723), -'BQ': (-63.14322, 17.566), -'BR': (-45.77065, -9.58685), -'BS': (-77.60916, 23.8745), -'BT': (90.01350, 27.28137), -'BV': (3.35744, -54.4215), -'BW': (23.51505, -23.48391), -'BY': (26.77259, 53.15885), -'BZ': (-88.63489, 16.33951), -'CA': (-107.74817, 67.12612), -'CC': (96.84420, -12.01734), -'CD': (24.09544, -1.67713), -'CF': (22.58701, 5.98438), -'CG': (15.78875, 0.40388), -'CH': (7.65705, 46.57446), -'CI': (-6.31190, 6.62783), -'CK': (-159.77835, -21.23349), -'CL': (-70.41790, -53.77189), -'CM': (13.26022, 5.94519), -'CN': (96.44285, 38.04260), -'CO': (-72.52951, 2.45174), -'CR': (-83.83314, 9.93514), -'CU': (-80.81673, 21.88852), -'CV': (-24.50810, 14.929), -'CW': (-68.96409, 12.1845), -'CX': (105.62411, -10.48417), -'CY': (32.95922, 35.37010), -'CZ': (16.32098, 49.50692), -'DE': (9.30716, 50.21289), -'DJ': (42.96904, 11.41542), -'DK': (9.18490, 55.98916), -'DM': (-61.00358, 15.65470), -'DO': (-69.62855, 18.58841), -'DZ': (4.24749, 25.79721), -'EC': (-77.45831, -0.98284), -'EE': (23.94288, 58.43952), -'EG': (28.95293, 28.17718), -'EH': (-13.69031, 25.01241), -'ER': (39.01223, 14.96033), -'ES': (-2.59110, 38.79354), -'ET': (38.61697, 7.71399), -'FI': (26.89798, 63.56194), -'FJ': (177.91853, -17.74237), -'FK': (-58.99044, -51.34509), -'FM': (151.95358, 8.5045), -'FO': (-6.60483, 62.10000), -'FR': (0.28410, 47.51045), -'GA': (10.81070, -0.07429), -'GB': (-0.92823, 52.01618), -'GD': (-61.64524, 12.191), -'GE': (44.16664, 42.00385), -'GF': (-53.46524, 3.56188), -'GG': (-2.50580, 49.58543), -'GH': (-0.46348, 7.16051), -'GI': (-5.32053, 36.11066), -'GL': (-33.85511, 74.66355), -'GM': (-16.40960, 13.25), -'GN': (-13.83940, 10.96291), -'GP': (-61.68712, 16.23049), -'GQ': (10.23973, 1.43119), -'GR': (23.17850, 39.06206), -'GS': (-36.49430, -54.43067), -'GT': (-90.74368, 15.20428), -'GU': (144.73362, 13.44413), -'GW': (-14.83525, 11.92486), -'GY': (-58.45167, 5.73698), -'HK': (114.18577, 22.34923), -'HM': (73.68230, -53.22105), -'HN': (-86.95414, 15.23820), -'HR': (17.49966, 45.52689), -'HT': (-73.51925, 18.32492), -'HU': (20.35362, 47.51721), -'ID': (123.34505, -0.83791), -'IE': (-9.00520, 52.87725), -'IL': (35.46314, 32.86165), -'IM': (-4.86740, 54.023), -'IN': (88.67620, 27.86155), -'IO': (71.42743, -6.14349), -'IQ': (42.58109, 34.26103), -'IR': (56.09355, 30.46751), -'IS': (-17.51785, 64.71687), -'IT': (10.42639, 44.87904), -'JE': (-2.19261, 49.12458), -'JM': (-76.84020, 18.3935), -'JO': (36.55552, 30.75741), -'JP': (138.72531, 35.92099), -'KE': (36.90602, 1.08512), -'KG': (76.15571, 41.66497), -'KH': (104.31901, 12.95555), -'KI': (173.63353, 0.139), -'KM': (44.31474, -12.241), -'KN': (-62.69379, 17.2555), -'KP': (126.65575, 39.64575), -'KR': (127.27740, 36.41388), -'KW': (47.30684, 29.69180), -'KY': (-81.07455, 19.29949), -'KZ': (72.00811, 49.88855), -'LA': (102.44391, 19.81609), -'LB': (35.48464, 33.41766), -'LC': (-60.97894, 13.891), -'LI': (9.54693, 47.15934), -'LK': (80.38520, 8.41649), -'LR': (-11.16960, 4.04122), -'LS': (28.66984, -29.94538), -'LT': (24.51735, 55.49293), -'LU': (6.08649, 49.81533), -'LV': (23.51033, 56.67144), -'LY': (15.36841, 28.12177), -'MA': (-4.03061, 33.21696), -'MC': (7.47743, 43.62917), -'MD': (29.61725, 46.66517), -'ME': (19.72291, 43.02441), -'MF': (-63.06666, 18.08102), -'MG': (45.86378, -20.50245), -'MH': (171.94982, 5.983), -'MK': (21.42108, 41.08980), -'ML': (-1.93310, 16.46993), -'MM': (95.54624, 21.09620), -'MN': (99.81138, 48.18615), -'MO': (113.56441, 22.16209), -'MP': (145.21345, 14.14902), -'MQ': (-60.81128, 14.43706), -'MR': (-9.42324, 22.59251), -'MS': (-62.19455, 16.745), -'MT': (14.38363, 35.94467), -'MU': (57.55121, -20.41), -'MV': (73.39292, 4.19375), -'MW': (33.95722, -12.28218), -'MX': (-105.89221, 25.86826), -'MY': (112.71154, 2.10098), -'MZ': (37.58689, -13.72682), -'NA': (16.68569, -21.46572), -'NC': (164.95322, -20.38889), -'NE': (10.06041, 19.08273), -'NF': (167.95718, -29.0645), -'NG': (10.17781, 10.17804), -'NI': (-85.87974, 13.21715), -'NL': (-68.57062, 12.041), -'NO': (23.11556, 70.09934), -'NP': (83.36259, 28.13107), -'NR': (166.93479, -0.5275), -'NU': (-169.84873, -19.05305), -'NZ': (167.97209, -45.13056), -'OM': (56.86055, 20.47413), -'PA': (-79.40160, 8.80656), -'PE': (-78.66540, -7.54711), -'PF': (-145.05719, -16.70862), -'PG': (146.64600, -7.37427), -'PH': (121.48359, 15.09965), -'PK': (72.11347, 31.14629), -'PL': (17.88136, 52.77182), -'PM': (-56.19515, 46.78324), -'PN': (-130.10642, -25.06955), -'PR': (-65.88755, 18.37169), -'PS': (35.39801, 32.24773), -'PT': (-8.45743, 40.11154), -'PW': (134.49645, 7.3245), -'PY': (-59.51787, -22.41281), -'QA': (51.49903, 24.99816), -'RE': (55.77345, -21.36388), -'RO': (26.37632, 45.36120), -'RS': (20.40371, 44.56413), -'RU': (116.44060, 59.06780), -'RW': (29.57882, -1.62404), -'SA': (47.73169, 22.43790), -'SB': (164.63894, -10.23606), -'SC': (46.36566, -9.454), -'SD': (28.14720, 14.56423), -'SE': (15.68667, 60.35568), -'SG': (103.84187, 1.304), -'SH': (-12.28155, -37.11546), -'SI': (14.04738, 46.39085), -'SJ': (15.27552, 79.23365), -'SK': (20.41603, 48.86970), -'SL': (-11.47773, 8.78156), -'SM': (12.46062, 43.94279), -'SN': (-15.37111, 14.99477), -'SO': (46.93383, 9.34094), -'SR': (-55.42864, 4.56985), -'SS': (28.13573, 8.50933), -'ST': (6.61025, 0.2215), -'SV': (-89.36665, 13.43072), -'SX': (-63.15393, 17.9345), -'SY': (38.15513, 35.34221), -'SZ': (31.78263, -26.14244), -'TC': (-71.32554, 21.35), -'TD': (17.42092, 13.46223), -'TF': (137.5, -67.5), -'TG': (1.06983, 7.87677), -'TH': (102.00877, 16.42310), -'TJ': (71.91349, 39.01527), -'TK': (-171.82603, -9.20990), -'TL': (126.22520, -8.72636), -'TM': (57.71603, 39.92534), -'TN': (9.04958, 34.84199), -'TO': (-176.99320, -23.11104), -'TR': (32.82002, 39.86350), -'TT': (-60.70793, 11.1385), -'TV': (178.77499, -9.41685), -'TW': (120.30074, 23.17002), -'TZ': (33.53892, -5.01840), -'UA': (33.44335, 49.30619), -'UG': (32.96523, 2.08584), -'UM': (-169.50993, 16.74605), -'US': (-116.39535, 40.71379), -'UY': (-56.46505, -33.62658), -'UZ': (61.35529, 42.96107), -'VA': (12.33197, 42.04931), -'VC': (-61.09905, 13.316), -'VE': (-64.88323, 7.69849), -'VG': (-64.62479, 18.419), -'VI': (-64.88950, 18.32263), -'VN': (104.20179, 10.27644), -'VU': (167.31919, -15.88687), -'WF': (-176.20781, -13.28535), -'WS': (-172.10966, -13.85093), -'YE': (45.94562, 16.16338), -'YT': (44.93774, -12.60882), -'ZA': (23.19488, -30.43276), -'ZM': (26.38618, -14.39966), -'ZW': (30.12419, -19.86907) -} + # Country aliases + 'AD': (1.58972, 42.54241), + 'AE': (54.61589, 24.82431), + 'AF': (65.90264, 34.84708), + 'AG': (-61.72430, 17.069), + 'AI': (-63.10571, 18.25461), + 'AL': (19.84941, 40.21232), + 'AM': (44.64229, 40.37821), + 'AO': (16.21924, -12.77014), + 'AQ': (44.99999, -75.65695), + 'AR': (-61.10759, -34.37615), + 'AS': (-170.68470, -14.29307), + 'AT': (14.25747, 47.36542), + 'AU': (138.23155, -23.72068), + 'AW': (-69.98255, 12.555), + 'AX': (19.91839, 59.81682), + 'AZ': (48.38555, 40.61639), + 'BA': (17.18514, 44.25582), + 'BB': (-59.53342, 13.19), + 'BD': (89.75989, 24.34205), + 'BE': (4.90078, 50.34682), + 'BF': (-0.56743, 11.90471), + 'BG': (24.80616, 43.09859), + 'BH': (50.52032, 25.94685), + 'BI': (29.54561, -2.99057), + 'BJ': (2.70062, 10.02792), + 'BL': (-62.79349, 17.907), + 'BM': (-64.77406, 32.30199), + 'BN': (114.52196, 4.28638), + 'BO': (-62.02473, -17.77723), + 'BQ': (-63.14322, 17.566), + 'BR': (-45.77065, -9.58685), + 'BS': (-77.60916, 23.8745), + 'BT': (90.01350, 27.28137), + 'BV': (3.35744, -54.4215), + 'BW': (23.51505, -23.48391), + 'BY': (26.77259, 53.15885), + 'BZ': (-88.63489, 16.33951), + 'CA': (-107.74817, 67.12612), + 'CC': (96.84420, -12.01734), + 'CD': (24.09544, -1.67713), + 'CF': (22.58701, 5.98438), + 'CG': (15.78875, 0.40388), + 'CH': (7.65705, 46.57446), + 'CI': (-6.31190, 6.62783), + 'CK': (-159.77835, -21.23349), + 'CL': (-70.41790, -53.77189), + 'CM': (13.26022, 5.94519), + 'CN': (96.44285, 38.04260), + 'CO': (-72.52951, 2.45174), + 'CR': (-83.83314, 9.93514), + 'CU': (-80.81673, 21.88852), + 'CV': (-24.50810, 14.929), + 'CW': (-68.96409, 12.1845), + 'CX': (105.62411, -10.48417), + 'CY': (32.95922, 35.37010), + 'CZ': (16.32098, 49.50692), + 'DE': (9.30716, 50.21289), + 'DJ': (42.96904, 11.41542), + 'DK': (9.18490, 55.98916), + 'DM': (-61.00358, 15.65470), + 'DO': (-69.62855, 18.58841), + 'DZ': (4.24749, 25.79721), + 'EC': (-77.45831, -0.98284), + 'EE': (23.94288, 58.43952), + 'EG': (28.95293, 28.17718), + 'EH': (-13.69031, 25.01241), + 'ER': (39.01223, 14.96033), + 'ES': (-2.59110, 38.79354), + 'ET': (38.61697, 7.71399), + 'FI': (26.89798, 63.56194), + 'FJ': (177.91853, -17.74237), + 'FK': (-58.99044, -51.34509), + 'FM': (151.95358, 8.5045), + 'FO': (-6.60483, 62.10000), + 'FR': (0.28410, 47.51045), + 'GA': (10.81070, -0.07429), + 'GB': (-0.92823, 52.01618), + 'GD': (-61.64524, 12.191), + 'GE': (44.16664, 42.00385), + 'GF': (-53.46524, 3.56188), + 'GG': (-2.50580, 49.58543), + 'GH': (-0.46348, 7.16051), + 'GI': (-5.32053, 36.11066), + 'GL': (-33.85511, 74.66355), + 'GM': (-16.40960, 13.25), + 'GN': (-13.83940, 10.96291), + 'GP': (-61.68712, 16.23049), + 'GQ': (10.23973, 1.43119), + 'GR': (23.17850, 39.06206), + 'GS': (-36.49430, -54.43067), + 'GT': (-90.74368, 15.20428), + 'GU': (144.73362, 13.44413), + 'GW': (-14.83525, 11.92486), + 'GY': (-58.45167, 5.73698), + 'HK': (114.18577, 22.34923), + 'HM': (73.68230, -53.22105), + 'HN': (-86.95414, 15.23820), + 'HR': (17.49966, 45.52689), + 'HT': (-73.51925, 18.32492), + 'HU': (20.35362, 47.51721), + 'ID': (123.34505, -0.83791), + 'IE': (-9.00520, 52.87725), + 'IL': (35.46314, 32.86165), + 'IM': (-4.86740, 54.023), + 'IN': (88.67620, 27.86155), + 'IO': (71.42743, -6.14349), + 'IQ': (42.58109, 34.26103), + 'IR': (56.09355, 30.46751), + 'IS': (-17.51785, 64.71687), + 'IT': (10.42639, 44.87904), + 'JE': (-2.19261, 49.12458), + 'JM': (-76.84020, 18.3935), + 'JO': (36.55552, 30.75741), + 'JP': (138.72531, 35.92099), + 'KE': (36.90602, 1.08512), + 'KG': (76.15571, 41.66497), + 'KH': (104.31901, 12.95555), + 'KI': (173.63353, 0.139), + 'KM': (44.31474, -12.241), + 'KN': (-62.69379, 17.2555), + 'KP': (126.65575, 39.64575), + 'KR': (127.27740, 36.41388), + 'KW': (47.30684, 29.69180), + 'KY': (-81.07455, 19.29949), + 'KZ': (72.00811, 49.88855), + 'LA': (102.44391, 19.81609), + 'LB': (35.48464, 33.41766), + 'LC': (-60.97894, 13.891), + 'LI': (9.54693, 47.15934), + 'LK': (80.38520, 8.41649), + 'LR': (-11.16960, 4.04122), + 'LS': (28.66984, -29.94538), + 'LT': (24.51735, 55.49293), + 'LU': (6.08649, 49.81533), + 'LV': (23.51033, 56.67144), + 'LY': (15.36841, 28.12177), + 'MA': (-4.03061, 33.21696), + 'MC': (7.47743, 43.62917), + 'MD': (29.61725, 46.66517), + 'ME': (19.72291, 43.02441), + 'MF': (-63.06666, 18.08102), + 'MG': (45.86378, -20.50245), + 'MH': (171.94982, 5.983), + 'MK': (21.42108, 41.08980), + 'ML': (-1.93310, 16.46993), + 'MM': (95.54624, 21.09620), + 'MN': (99.81138, 48.18615), + 'MO': (113.56441, 22.16209), + 'MP': (145.21345, 14.14902), + 'MQ': (-60.81128, 14.43706), + 'MR': (-9.42324, 22.59251), + 'MS': (-62.19455, 16.745), + 'MT': (14.38363, 35.94467), + 'MU': (57.55121, -20.41), + 'MV': (73.39292, 4.19375), + 'MW': (33.95722, -12.28218), + 'MX': (-105.89221, 25.86826), + 'MY': (112.71154, 2.10098), + 'MZ': (37.58689, -13.72682), + 'NA': (16.68569, -21.46572), + 'NC': (164.95322, -20.38889), + 'NE': (10.06041, 19.08273), + 'NF': (167.95718, -29.0645), + 'NG': (10.17781, 10.17804), + 'NI': (-85.87974, 13.21715), + 'NL': (-68.57062, 12.041), + 'NO': (23.11556, 70.09934), + 'NP': (83.36259, 28.13107), + 'NR': (166.93479, -0.5275), + 'NU': (-169.84873, -19.05305), + 'NZ': (167.97209, -45.13056), + 'OM': (56.86055, 20.47413), + 'PA': (-79.40160, 8.80656), + 'PE': (-78.66540, -7.54711), + 'PF': (-145.05719, -16.70862), + 'PG': (146.64600, -7.37427), + 'PH': (121.48359, 15.09965), + 'PK': (72.11347, 31.14629), + 'PL': (17.88136, 52.77182), + 'PM': (-56.19515, 46.78324), + 'PN': (-130.10642, -25.06955), + 'PR': (-65.88755, 18.37169), + 'PS': (35.39801, 32.24773), + 'PT': (-8.45743, 40.11154), + 'PW': (134.49645, 7.3245), + 'PY': (-59.51787, -22.41281), + 'QA': (51.49903, 24.99816), + 'RE': (55.77345, -21.36388), + 'RO': (26.37632, 45.36120), + 'RS': (20.40371, 44.56413), + 'RU': (116.44060, 59.06780), + 'RW': (29.57882, -1.62404), + 'SA': (47.73169, 22.43790), + 'SB': (164.63894, -10.23606), + 'SC': (46.36566, -9.454), + 'SD': (28.14720, 14.56423), + 'SE': (15.68667, 60.35568), + 'SG': (103.84187, 1.304), + 'SH': (-12.28155, -37.11546), + 'SI': (14.04738, 46.39085), + 'SJ': (15.27552, 79.23365), + 'SK': (20.41603, 48.86970), + 'SL': (-11.47773, 8.78156), + 'SM': (12.46062, 43.94279), + 'SN': (-15.37111, 14.99477), + 'SO': (46.93383, 9.34094), + 'SR': (-55.42864, 4.56985), + 'SS': (28.13573, 8.50933), + 'ST': (6.61025, 0.2215), + 'SV': (-89.36665, 13.43072), + 'SX': (-63.15393, 17.9345), + 'SY': (38.15513, 35.34221), + 'SZ': (31.78263, -26.14244), + 'TC': (-71.32554, 21.35), + 'TD': (17.42092, 13.46223), + 'TF': (137.5, -67.5), + 'TG': (1.06983, 7.87677), + 'TH': (102.00877, 16.42310), + 'TJ': (71.91349, 39.01527), + 'TK': (-171.82603, -9.20990), + 'TL': (126.22520, -8.72636), + 'TM': (57.71603, 39.92534), + 'TN': (9.04958, 34.84199), + 'TO': (-176.99320, -23.11104), + 'TR': (32.82002, 39.86350), + 'TT': (-60.70793, 11.1385), + 'TV': (178.77499, -9.41685), + 'TW': (120.30074, 23.17002), + 'TZ': (33.53892, -5.01840), + 'UA': (33.44335, 49.30619), + 'UG': (32.96523, 2.08584), + 'UM': (-169.50993, 16.74605), + 'US': (-116.39535, 40.71379), + 'UY': (-56.46505, -33.62658), + 'UZ': (61.35529, 42.96107), + 'VA': (12.33197, 42.04931), + 'VC': (-61.09905, 13.316), + 'VE': (-64.88323, 7.69849), + 'VG': (-64.62479, 18.419), + 'VI': (-64.88950, 18.32263), + 'VN': (104.20179, 10.27644), + 'VU': (167.31919, -15.88687), + 'WF': (-176.20781, -13.28535), + 'WS': (-172.10966, -13.85093), + 'YE': (45.94562, 16.16338), + 'YT': (44.93774, -12.60882), + 'ZA': (23.19488, -30.43276), + 'ZM': (26.38618, -14.39966), + 'ZW': (30.12419, -19.86907) + } diff --git a/test/bdd/steps/geometry_factory.py b/test/bdd/steps/geometry_factory.py index 19c0406c..504227b3 100644 --- a/test/bdd/steps/geometry_factory.py +++ b/test/bdd/steps/geometry_factory.py @@ -2,13 +2,11 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. -from pathlib import Path -import os - from steps.geometry_alias import ALIASES + class GeometryFactory: """ Provides functions to create geometries from coordinates and data grids. """ @@ -47,7 +45,6 @@ class GeometryFactory: return "ST_SetSRID('{}'::geometry, 4326)".format(out) - def mk_wkt_point(self, point): """ Parse a point description. The point may either consist of 'x y' coordinates or a number @@ -65,7 +62,6 @@ class GeometryFactory: assert pt is not None, "Scenario error: Point '{}' not found in grid".format(geom) return "{} {}".format(*pt) - def mk_wkt_points(self, geom): """ Parse a list of points. The list must be a comma-separated list of points. Points @@ -73,7 +69,6 @@ class GeometryFactory: """ return ','.join([self.mk_wkt_point(x) for x in geom.split(',')]) - def set_grid(self, lines, grid_step, origin=(0.0, 0.0)): """ Replace the grid with one from the given lines. """ @@ -87,7 +82,6 @@ class GeometryFactory: x += grid_step y += grid_step - def grid_node(self, nodeid): """ Get the coordinates for the given grid node. """ diff --git a/test/bdd/steps/http_responses.py b/test/bdd/steps/http_responses.py index c28c4e1c..f803a45f 100644 --- a/test/bdd/steps/http_responses.py +++ b/test/bdd/steps/http_responses.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2023 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Classes wrapping HTTP responses from the Nominatim API. @@ -45,7 +45,6 @@ class GenericResponse: else: self.result = [self.result] - def _parse_geojson(self): self._parse_json() if self.result: @@ -76,7 +75,6 @@ class GenericResponse: new['__' + k] = v self.result.append(new) - def _parse_geocodejson(self): self._parse_geojson() if self.result: @@ -87,7 +85,6 @@ class GenericResponse: inner = r.pop('geocoding') r.update(inner) - def assert_address_field(self, idx, field, value): """ Check that result rows`idx` has a field `field` with value `value` in its address. If idx is None, then all results are checked. @@ -103,7 +100,6 @@ class GenericResponse: address = self.result[idx]['address'] self.check_row_field(idx, field, value, base=address) - def match_row(self, row, context=None, field=None): """ Match the result fields against the given behave table row. """ @@ -139,7 +135,6 @@ class GenericResponse: else: self.check_row_field(i, name, Field(value), base=subdict) - def check_row(self, idx, check, msg): """ Assert for the condition 'check' and print 'msg' on fail together with the contents of the failing result. @@ -154,7 +149,6 @@ class GenericResponse: assert check, _RowError(self.result[idx]) - def check_row_field(self, idx, field, expected, base=None): """ Check field 'field' of result 'idx' for the expected value and print a meaningful error if the condition fails. @@ -172,7 +166,6 @@ class GenericResponse: f"\nBad value for field '{field}'. Expected: {expected}, got: {value}") - class SearchResponse(GenericResponse): """ Specialised class for search and lookup responses. Transforms the xml response in a format similar to json. @@ -240,7 +233,8 @@ class ReverseResponse(GenericResponse): assert 'namedetails' not in self.result[0], "More than one namedetails in result" self.result[0]['namedetails'] = {} for tag in child: - assert len(tag) == 0, f"Namedetails element '{tag.attrib['desc']}' has subelements" + assert len(tag) == 0, \ + f"Namedetails element '{tag.attrib['desc']}' has subelements" self.result[0]['namedetails'][tag.attrib['desc']] = tag.text elif child.tag == 'geokml': assert 'geokml' not in self.result[0], "More than one geokml in result" diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index ba19bb48..45d42307 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -2,10 +2,9 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. from pathlib import Path -import importlib import tempfile import psycopg @@ -13,10 +12,9 @@ from psycopg import sql as pysql from nominatim_db import cli from nominatim_db.config import Configuration -from nominatim_db.db.connection import Connection, register_hstore, execute_scalar -from nominatim_db.tools import refresh +from nominatim_db.db.connection import register_hstore, execute_scalar from nominatim_db.tokenizer import factory as tokenizer_factory -from steps.utils import run_script + class NominatimEnvironment: """ Collects all functions for the execution of Nominatim functions. @@ -62,7 +60,6 @@ class NominatimEnvironment: dbargs['password'] = self.db_pass return psycopg.connect(**dbargs) - def write_nominatim_config(self, dbname): """ Set up a custom test configuration that connects to the given database. This sets up the environment variables so that they can @@ -101,7 +98,6 @@ class NominatimEnvironment: self.website_dir = tempfile.TemporaryDirectory() - def get_test_config(self): cfg = Configuration(Path(self.website_dir.name), environ=self.test_env) return cfg @@ -122,14 +118,13 @@ class NominatimEnvironment: return dsn - def db_drop_database(self, name): """ Drop the database with the given name. """ with self.connect_database('postgres') as conn: conn.autocommit = True conn.execute(pysql.SQL('DROP DATABASE IF EXISTS') - + pysql.Identifier(name)) + + pysql.Identifier(name)) def setup_template_db(self): """ Setup a template database that already contains common test data. @@ -153,13 +148,12 @@ class NominatimEnvironment: '--osm2pgsql-cache', '1', '--ignore-errors', '--offline', '--index-noanalyse') - except: + except: # noqa: E722 self.db_drop_database(self.template_db) raise self.run_nominatim('refresh', '--functions') - def setup_api_db(self): """ Setup a test against the API test database. """ @@ -184,13 +178,12 @@ class NominatimEnvironment: csv_path = str(testdata / 'full_en_phrases_test.csv') self.run_nominatim('special-phrases', '--import-from-csv', csv_path) - except: + except: # noqa: E722 self.db_drop_database(self.api_test_db) raise tokenizer_factory.get_tokenizer_for_db(self.get_test_config()) - def setup_unknown_db(self): """ Setup a test against a non-existing database. """ @@ -213,7 +206,7 @@ class NominatimEnvironment: with self.connect_database(self.template_db) as conn: conn.autocommit = True conn.execute(pysql.SQL('DROP DATABASE IF EXISTS') - + pysql.Identifier(self.test_db)) + + pysql.Identifier(self.test_db)) conn.execute(pysql.SQL('CREATE DATABASE {} TEMPLATE = {}').format( pysql.Identifier(self.test_db), pysql.Identifier(self.template_db))) @@ -250,7 +243,6 @@ class NominatimEnvironment: return False - def reindex_placex(self, db): """ Run the indexing step until all data in the placex has been processed. Indexing during updates can produce more data @@ -259,7 +251,6 @@ class NominatimEnvironment: """ self.run_nominatim('index') - def run_nominatim(self, *cmdline): """ Run the nominatim command-line tool via the library. """ @@ -270,7 +261,6 @@ class NominatimEnvironment: cli_args=cmdline, environ=self.test_env) - def copy_from_place(self, db): """ Copy data from place to the placex and location_property_osmline tables invoking the appropriate triggers. @@ -293,7 +283,6 @@ class NominatimEnvironment: and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'""") - def create_api_request_func_starlette(self): import nominatim_api.server.starlette.server from asgi_lifespan import LifespanManager @@ -311,7 +300,6 @@ class NominatimEnvironment: return _request - def create_api_request_func_falcon(self): import nominatim_api.server.falcon.server import falcon.testing @@ -326,6 +314,3 @@ class NominatimEnvironment: return response.text, response.status_code return _request - - - diff --git a/test/bdd/steps/place_inserter.py b/test/bdd/steps/place_inserter.py index bd6c47df..dcd2baec 100644 --- a/test/bdd/steps/place_inserter.py +++ b/test/bdd/steps/place_inserter.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Helper classes for filling the place table. @@ -10,12 +10,13 @@ Helper classes for filling the place table. import random import string + class PlaceColumn: """ Helper class to collect contents from a behave table row and insert it into the place table. """ def __init__(self, context): - self.columns = {'admin_level' : 15} + self.columns = {'admin_level': 15} self.context = context self.geometry = None @@ -98,7 +99,7 @@ class PlaceColumn: """ Issue a delete for the given OSM object. """ cursor.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s', - (self.columns['osm_type'] , self.columns['osm_id'])) + (self.columns['osm_type'], self.columns['osm_id'])) def db_insert(self, cursor): """ Insert the collected data into the database. diff --git a/test/bdd/steps/steps_api_queries.py b/test/bdd/steps/steps_api_queries.py index 4d15381d..de38549e 100644 --- a/test/bdd/steps/steps_api_queries.py +++ b/test/bdd/steps/steps_api_queries.py @@ -2,20 +2,16 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Steps that run queries against the API. """ from pathlib import Path -import json -import os import re import logging import asyncio import xml.etree.ElementTree as ET -from urllib.parse import urlencode -from utils import run_script from http_responses import GenericResponse, SearchResponse, ReverseResponse, StatusResponse from check_functions import Bbox, check_for_attributes from table_compare import NominatimID @@ -68,7 +64,7 @@ def send_api_query(endpoint, params, fmt, context): getattr(context, 'http_headers', {}))) -@given(u'the HTTP header') +@given('the HTTP header') def add_http_header(context): if not hasattr(context, 'http_headers'): context.http_headers = {} @@ -77,7 +73,7 @@ def add_http_header(context): context.http_headers[h] = context.table[0][h] -@when(u'sending (?P\S+ )?search query "(?P.*)"(?P with address)?') +@when(r'sending (?P\S+ )?search query "(?P.*)"(?P with address)?') def website_search_request(context, fmt, query, addr): params = {} if query: @@ -90,7 +86,7 @@ def website_search_request(context, fmt, query, addr): context.response = SearchResponse(outp, fmt or 'json', status) -@when('sending v1/reverse at (?P[\d.-]*),(?P[\d.-]*)(?: with format (?P.+))?') +@when(r'sending v1/reverse at (?P[\d.-]*),(?P[\d.-]*)(?: with format (?P.+))?') def api_endpoint_v1_reverse(context, lat, lon, fmt): params = {} if lat is not None: @@ -106,7 +102,7 @@ def api_endpoint_v1_reverse(context, lat, lon, fmt): context.response = ReverseResponse(outp, fmt or 'xml', status) -@when('sending v1/reverse N(?P\d+)(?: with format (?P.+))?') +@when(r'sending v1/reverse N(?P\d+)(?: with format (?P.+))?') def api_endpoint_v1_reverse_from_node(context, nodeid, fmt): params = {} params['lon'], params['lat'] = (f'{c:f}' for c in context.osm.grid_node(int(nodeid))) @@ -115,7 +111,7 @@ def api_endpoint_v1_reverse_from_node(context, nodeid, fmt): context.response = ReverseResponse(outp, fmt or 'xml', status) -@when(u'sending (?P\S+ )?details query for (?P.*)') +@when(r'sending (?P\S+ )?details query for (?P.*)') def website_details_request(context, fmt, query): params = {} if query[0] in 'NWR': @@ -130,38 +126,45 @@ def website_details_request(context, fmt, query): context.response = GenericResponse(outp, fmt or 'json', status) -@when(u'sending (?P\S+ )?lookup query for (?P.*)') + +@when(r'sending (?P\S+ )?lookup query for (?P.*)') def website_lookup_request(context, fmt, query): - params = { 'osm_ids' : query } + params = {'osm_ids': query} outp, status = send_api_query('lookup', params, fmt, context) context.response = SearchResponse(outp, fmt or 'xml', status) -@when(u'sending (?P\S+ )?status query') + +@when(r'sending (?P\S+ )?status query') def website_status_request(context, fmt): params = {} outp, status = send_api_query('status', params, fmt, context) context.response = StatusResponse(outp, fmt or 'text', status) -@step(u'(?Pless than|more than|exactly|at least|at most) (?P\d+) results? (?:is|are) returned') + +@step(r'(?Pless than|more than|exactly|at least|at most) ' + r'(?P\d+) results? (?:is|are) returned') def validate_result_number(context, operator, number): context.execute_steps("Then a HTTP 200 is returned") numres = len(context.response.result) assert compare(operator, numres, int(number)), \ f"Bad number of results: expected {operator} {number}, got {numres}." -@then(u'a HTTP (?P\d+) is returned') + +@then(r'a HTTP (?P\d+) is returned') def check_http_return_status(context, status): assert context.response.errorcode == int(status), \ f"Return HTTP status is {context.response.errorcode}."\ f" Full response:\n{context.response.page}" -@then(u'the page contents equals "(?P.+)"') + +@then(r'the page contents equals "(?P.+)"') def check_page_content_equals(context, text): assert context.response.page == text -@then(u'the result is valid (?P\w+)') + +@then(r'the result is valid (?P\w+)') def step_impl(context, fmt): context.execute_steps("Then a HTTP 200 is returned") if fmt.strip() == 'html': @@ -178,7 +181,7 @@ def step_impl(context, fmt): assert context.response.format == fmt -@then(u'a (?P\w+) user error is returned') +@then(r'a (?P\w+) user error is returned') def check_page_error(context, fmt): context.execute_steps("Then a HTTP 400 is returned") assert context.response.format == fmt @@ -188,32 +191,34 @@ def check_page_error(context, fmt): else: assert re.search(r'({"error":)', context.response.page, re.DOTALL) is not None -@then(u'result header contains') + +@then('result header contains') def check_header_attr(context): context.execute_steps("Then a HTTP 200 is returned") for line in context.table: assert line['attr'] in context.response.header, \ - f"Field '{line['attr']}' missing in header. Full header:\n{context.response.header}" + f"Field '{line['attr']}' missing in header. " \ + f"Full header:\n{context.response.header}" value = context.response.header[line['attr']] assert re.fullmatch(line['value'], value) is not None, \ f"Attribute '{line['attr']}': expected: '{line['value']}', got '{value}'" -@then(u'result header has (?Pnot )?attributes (?P.*)') +@then('result header has (?Pnot )?attributes (?P.*)') def check_header_no_attr(context, neg, attrs): check_for_attributes(context.response.header, attrs, 'absent' if neg else 'present') -@then(u'results contain(?: in field (?P.*))?') -def step_impl(context, field): +@then(r'results contain(?: in field (?P.*))?') +def results_contain_in_field(context, field): context.execute_steps("then at least 1 result is returned") for line in context.table: context.response.match_row(line, context=context, field=field) -@then(u'result (?P\d+ )?has (?Pnot )?attributes (?P.*)') +@then(r'result (?P\d+ )?has (?Pnot )?attributes (?P.*)') def validate_attributes(context, lid, neg, attrs): for i in make_todo_list(context, lid): check_for_attributes(context.response.result[i], attrs, @@ -221,7 +226,7 @@ def validate_attributes(context, lid, neg, attrs): @then(u'result addresses contain') -def step_impl(context): +def result_addresses_contain(context): context.execute_steps("then at least 1 result is returned") for line in context.table: @@ -231,8 +236,9 @@ def step_impl(context): if name != 'ID': context.response.assert_address_field(idx, name, value) -@then(u'address of result (?P\d+) has(?P no)? types (?P.*)') -def check_address(context, lid, neg, attrs): + +@then(r'address of result (?P\d+) has(?P no)? types (?P.*)') +def check_address_has_types(context, lid, neg, attrs): context.execute_steps(f"then more than {lid} results are returned") addr_parts = context.response.result[int(lid)]['address'] @@ -243,7 +249,8 @@ def check_address(context, lid, neg, attrs): else: assert attr in addr_parts -@then(u'address of result (?P\d+) (?Pis|contains)') + +@then(r'address of result (?P\d+) (?Pis|contains)') def check_address(context, lid, complete): context.execute_steps(f"then more than {lid} results are returned") @@ -258,7 +265,7 @@ def check_address(context, lid, complete): assert len(addr_parts) == 0, f"Additional address parts found: {addr_parts!s}" -@then(u'result (?P\d+ )?has bounding box in (?P[\d,.-]+)') +@then(r'result (?P\d+ )?has bounding box in (?P[\d,.-]+)') def check_bounding_box_in_area(context, lid, coords): expected = Bbox(coords) @@ -269,7 +276,7 @@ def check_bounding_box_in_area(context, lid, coords): f"Bbox is not contained in {expected}") -@then(u'result (?P\d+ )?has centroid in (?P[\d,.-]+)') +@then(r'result (?P\d+ )?has centroid in (?P[\d,.-]+)') def check_centroid_in_area(context, lid, coords): expected = Bbox(coords) @@ -280,7 +287,7 @@ def check_centroid_in_area(context, lid, coords): f"Centroid is not inside {expected}") -@then(u'there are(?P no)? duplicates') +@then('there are(?P no)? duplicates') def check_for_duplicates(context, neg): context.execute_steps("then at least 1 result is returned") @@ -298,4 +305,3 @@ def check_for_duplicates(context, neg): assert not has_dupe, f"Found duplicate for {dup}" else: assert has_dupe, "No duplicates found" - diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py index fb8431d5..8b62cbc6 100644 --- a/test/bdd/steps/steps_db_ops.py +++ b/test/bdd/steps/steps_db_ops.py @@ -2,9 +2,8 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. -import logging from itertools import chain import psycopg @@ -13,9 +12,9 @@ from psycopg import sql as pysql from place_inserter import PlaceColumn from table_compare import NominatimID, DBRow -from nominatim_db.indexer import indexer from nominatim_db.tokenizer import factory as tokenizer_factory + def check_database_integrity(context): """ Check some generic constraints on the tables. """ @@ -31,10 +30,9 @@ def check_database_integrity(context): cur.execute("SELECT count(*) FROM word WHERE word_token = ''") assert cur.fetchone()[0] == 0, "Empty word tokens found in word table" +# GIVEN ################################## -################################ GIVEN ################################## - @given("the (?Pnamed )?places") def add_data_to_place_table(context, named): """ Add entries into the place table. 'named places' makes sure that @@ -46,6 +44,7 @@ def add_data_to_place_table(context, named): PlaceColumn(context).add_row(row, named is not None).db_insert(cur) cur.execute('ALTER TABLE place ENABLE TRIGGER place_before_insert') + @given("the relations") def add_data_to_planet_relations(context): """ Add entries into the osm2pgsql relation middle table. This is needed @@ -77,9 +76,11 @@ def add_data_to_planet_relations(context): else: members = None - tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")]) + tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings + if h.startswith("tags+")]) - cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, parts, members, tags) + cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, + parts, members, tags) VALUES (%s, %s, %s, %s, %s, %s)""", (r['id'], last_node, last_way, parts, members, list(tags))) else: @@ -99,6 +100,7 @@ def add_data_to_planet_relations(context): (r['id'], psycopg.types.json.Json(tags), psycopg.types.json.Json(members))) + @given("the ways") def add_data_to_planet_ways(context): """ Add entries into the osm2pgsql way middle table. This is necessary for @@ -110,16 +112,18 @@ def add_data_to_planet_ways(context): json_tags = row is not None and row['value'] != '1' for r in context.table: if json_tags: - tags = psycopg.types.json.Json({h[5:]: r[h] for h in r.headings if h.startswith("tags+")}) + tags = psycopg.types.json.Json({h[5:]: r[h] for h in r.headings + if h.startswith("tags+")}) else: tags = list(chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")])) - nodes = [ int(x.strip()) for x in r['nodes'].split(',') ] + nodes = [int(x.strip()) for x in r['nodes'].split(',')] cur.execute("INSERT INTO planet_osm_ways (id, nodes, tags) VALUES (%s, %s, %s)", (r['id'], nodes, tags)) -################################ WHEN ################################## +# WHEN ################################## + @when("importing") def import_and_index_data_from_place_table(context): @@ -136,6 +140,7 @@ def import_and_index_data_from_place_table(context): # itself. context.log_capture.buffer.clear() + @when("updating places") def update_place_table(context): """ Update the place table with the given data. Also runs all triggers @@ -164,6 +169,7 @@ def update_postcodes(context): """ context.nominatim.run_nominatim('refresh', '--postcodes') + @when("marking for delete (?P.*)") def delete_places(context, oids): """ Remove entries from the place table. Multiple ids may be given @@ -184,7 +190,8 @@ def delete_places(context, oids): # itself. context.log_capture.buffer.clear() -################################ THEN ################################## +# THEN ################################## + @then("(?Pplacex|place) contains(?P exactly)?") def check_place_contents(context, table, exact): @@ -201,7 +208,8 @@ def check_place_contents(context, table, exact): expected_content = set() for row in context.table: nid = NominatimID(row['object']) - query = 'SELECT *, ST_AsText(geometry) as geomtxt, ST_GeometryType(geometry) as geometrytype' + query = """SELECT *, ST_AsText(geometry) as geomtxt, + ST_GeometryType(geometry) as geometrytype """ if table == 'placex': query += ' ,ST_X(centroid) as cx, ST_Y(centroid) as cy' query += " FROM %s WHERE {}" % (table, ) @@ -261,17 +269,18 @@ def check_search_name_contents(context, exclude): if not exclude: assert len(tokens) >= len(items), \ - "No word entry found for {}. Entries found: {!s}".format(value, len(tokens)) + f"No word entry found for {value}. Entries found: {len(tokens)}" for word, token, wid in tokens: if exclude: assert wid not in res[name], \ - "Found term for {}/{}: {}".format(nid, name, wid) + "Found term for {}/{}: {}".format(nid, name, wid) else: assert wid in res[name], \ - "Missing term for {}/{}: {}".format(nid, name, wid) + "Missing term for {}/{}: {}".format(nid, name, wid) elif name != 'object': assert db_row.contains(name, value), db_row.assert_msg(name, value) + @then("search_name has no entry for (?P.*)") def check_search_name_has_entry(context, oid): """ Check that there is noentry in the search_name table for the given @@ -283,6 +292,7 @@ def check_search_name_has_entry(context, oid): assert cur.rowcount == 0, \ "Found {} entries for ID {}".format(cur.rowcount, oid) + @then("location_postcode contains exactly") def check_location_postcode(context): """ Check full contents for location_postcode table. Each row represents a table row @@ -294,21 +304,22 @@ def check_location_postcode(context): with context.db.cursor() as cur: cur.execute("SELECT *, ST_AsText(geometry) as geomtxt FROM location_postcode") assert cur.rowcount == len(list(context.table)), \ - "Postcode table has {} rows, expected {}.".format(cur.rowcount, len(list(context.table))) + "Postcode table has {cur.rowcount} rows, expected {len(list(context.table))}." results = {} for row in cur: key = (row['country_code'], row['postcode']) assert key not in results, "Postcode table has duplicate entry: {}".format(row) - results[key] = DBRow((row['country_code'],row['postcode']), row, context) + results[key] = DBRow((row['country_code'], row['postcode']), row, context) for row in context.table: - db_row = results.get((row['country'],row['postcode'])) + db_row = results.get((row['country'], row['postcode'])) assert db_row is not None, \ f"Missing row for country '{row['country']}' postcode '{row['postcode']}'." db_row.assert_row(row, ('country', 'postcode')) + @then("there are(?P no)? word tokens for postcodes (?P.*)") def check_word_table_for_postcodes(context, exclude, postcodes): """ Check that the tokenizer produces postcode tokens for the given @@ -333,7 +344,8 @@ def check_word_table_for_postcodes(context, exclude, postcodes): assert len(found) == 0, f"Unexpected postcodes: {found}" else: assert set(found) == set(plist), \ - f"Missing postcodes {set(plist) - set(found)}. Found: {found}" + f"Missing postcodes {set(plist) - set(found)}. Found: {found}" + @then("place_addressline contains") def check_place_addressline(context): @@ -352,11 +364,12 @@ def check_place_addressline(context): WHERE place_id = %s AND address_place_id = %s""", (pid, apid)) assert cur.rowcount > 0, \ - "No rows found for place %s and address %s" % (row['object'], row['address']) + f"No rows found for place {row['object']} and address {row['address']}." for res in cur: DBRow(nid, res, context).assert_row(row, ('address', 'object')) + @then("place_addressline doesn't contain") def check_place_addressline_exclude(context): """ Check that the place_addressline doesn't contain any entries for the @@ -371,9 +384,10 @@ def check_place_addressline_exclude(context): WHERE place_id = %s AND address_place_id = %s""", (pid, apid)) assert cur.rowcount == 0, \ - "Row found for place %s and address %s" % (row['object'], row['address']) + f"Row found for place {row['object']} and address {row['address']}." + -@then("W(?P\d+) expands to(?P no)? interpolation") +@then(r"W(?P\d+) expands to(?P no)? interpolation") def check_location_property_osmline(context, oid, neg): """ Check that the given way is present in the interpolation table. """ @@ -392,7 +406,7 @@ def check_location_property_osmline(context, oid, neg): for i in todo: row = context.table[i] if (int(row['start']) == res['startnumber'] - and int(row['end']) == res['endnumber']): + and int(row['end']) == res['endnumber']): todo.remove(i) break else: @@ -402,8 +416,9 @@ def check_location_property_osmline(context, oid, neg): assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}" + @then("location_property_osmline contains(?P exactly)?") -def check_place_contents(context, exact): +def check_osmline_contents(context, exact): """ Check contents of the interpolation table. Each row represents a table row and all data must match. Data not present in the expected table, may be arbitrary. The rows are identified via the 'object' column which must @@ -447,4 +462,3 @@ def check_place_contents(context, exact): assert expected_content == actual, \ f"Missing entries: {expected_content - actual}\n" \ f"Not expected in table: {actual - expected_content}" - diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py index e9c8ebe4..69f71994 100644 --- a/test/bdd/steps/steps_osm_data.py +++ b/test/bdd/steps/steps_osm_data.py @@ -14,6 +14,7 @@ from nominatim_db.tools.replication import run_osm2pgsql_updates from geometry_alias import ALIASES + def get_osm2pgsql_options(nominatim_env, fname, append): return dict(import_file=fname, osm2pgsql='osm2pgsql', @@ -25,8 +26,7 @@ def get_osm2pgsql_options(nominatim_env, fname, append): flatnode_file='', tablespaces=dict(slim_data='', slim_index='', main_data='', main_index=''), - append=append - ) + append=append) def write_opl_file(opl, grid): @@ -48,6 +48,7 @@ def write_opl_file(opl, grid): return fd.name + @given('the lua style file') def lua_style_file(context): """ Define a custom style file to use for the import. @@ -90,7 +91,7 @@ def define_node_grid(context, grid_step, origin): @when(u'loading osm data') def load_osm_file(context): """ - Load the given data into a freshly created test data using osm2pgsql. + Load the given data into a freshly created test database using osm2pgsql. No further indexing is done. The data is expected as attached text in OPL format. @@ -102,13 +103,14 @@ def load_osm_file(context): finally: os.remove(fname) - ### reintroduce the triggers/indexes we've lost by having osm2pgsql set up place again + # reintroduce the triggers/indexes we've lost by having osm2pgsql set up place again cur = context.db.cursor() cur.execute("""CREATE TRIGGER place_before_delete BEFORE DELETE ON place FOR EACH ROW EXECUTE PROCEDURE place_delete()""") cur.execute("""CREATE TRIGGER place_before_insert BEFORE INSERT ON place FOR EACH ROW EXECUTE PROCEDURE place_insert()""") - cur.execute("""CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type)""") + cur.execute("""CREATE UNIQUE INDEX idx_place_osm_unique ON place + USING btree(osm_id,osm_type,class,type)""") context.db.commit() @@ -132,6 +134,7 @@ def update_from_osm_file(context): finally: os.remove(fname) + @when('indexing') def index_database(context): """ diff --git a/test/bdd/steps/table_compare.py b/test/bdd/steps/table_compare.py index f0d27ba5..79c9186b 100644 --- a/test/bdd/steps/table_compare.py +++ b/test/bdd/steps/table_compare.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Functions to facilitate accessing and comparing the content of DB tables. @@ -16,6 +16,7 @@ from psycopg import sql as pysql ID_REGEX = re.compile(r"(?P[NRW])(?P\d+)(:(?P\w+))?") + class NominatimID: """ Splits a unique identifier for places into its components. As place_ids cannot be used for testing, we use a unique @@ -146,10 +147,10 @@ class DBRow: return str(actual) == expected def _compare_place_id(self, actual, expected): - if expected == '0': + if expected == '0': return actual == 0 - with self.context.db.cursor() as cur: + with self.context.db.cursor() as cur: return NominatimID(expected).get_place_id(cur) == actual def _has_centroid(self, expected): @@ -165,13 +166,15 @@ class DBRow: else: x, y = self.context.osm.grid_node(int(expected)) - return math.isclose(float(x), self.db_row['cx']) and math.isclose(float(y), self.db_row['cy']) + return math.isclose(float(x), self.db_row['cx']) \ + and math.isclose(float(y), self.db_row['cy']) def _has_geometry(self, expected): geom = self.context.osm.parse_geometry(expected) with self.context.db.cursor(row_factory=psycopg.rows.tuple_row) as cur: - cur.execute(pysql.SQL("""SELECT ST_Equals(ST_SnapToGrid({}, 0.00001, 0.00001), - ST_SnapToGrid(ST_SetSRID({}::geometry, 4326), 0.00001, 0.00001))""") + cur.execute(pysql.SQL(""" + SELECT ST_Equals(ST_SnapToGrid({}, 0.00001, 0.00001), + ST_SnapToGrid(ST_SetSRID({}::geometry, 4326), 0.00001, 0.00001))""") .format(pysql.SQL(geom), pysql.Literal(self.db_row['geomtxt']))) return cur.fetchone()[0] @@ -186,7 +189,8 @@ class DBRow: else: msg += " No such column." - return msg + "\nFull DB row: {}".format(json.dumps(dict(self.db_row), indent=4, default=str)) + return msg + "\nFull DB row: {}".format(json.dumps(dict(self.db_row), + indent=4, default=str)) def _get_actual(self, name): if '+' in name: diff --git a/test/bdd/steps/utils.py b/test/bdd/steps/utils.py index e789deff..19ce7928 100644 --- a/test/bdd/steps/utils.py +++ b/test/bdd/steps/utils.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Various smaller helps for step execution. @@ -12,6 +12,7 @@ import subprocess LOG = logging.getLogger(__name__) + def run_script(cmd, **kwargs): """ Run the given command, check that it is successful and output when necessary.