From 474aea61da51df47cfb5b3cc8f4fdff9b0cf23fc Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 30 Jul 2024 11:38:40 +0200 Subject: [PATCH] bdd tests: get rid of dependency on build path BDD tests will now use whatever osm2pgsql they find in the PATH. For testing against legacy tokenizer, use -DSERVER_MODULE_PATH to point to the module. This will no longer work out of the box. --- docs/develop/Development-Environment.md | 4 ++++ docs/develop/Testing.md | 3 +-- test/bdd/environment.py | 8 +++++--- test/bdd/steps/nominatim_environment.py | 19 ++++++------------- test/bdd/steps/steps_osm_data.py | 2 +- 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index a7c474e8..19948e7c 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -50,6 +50,10 @@ The documentation is built with mkdocs: * [mkdocstrings](https://mkdocstrings.github.io/) >= 0.25 * [mkdocs-material](https://squidfunk.github.io/mkdocs-material/) +Please be aware that tests always run against the globally installed +osm2pgsql, so you need to have this set up. If you want to test against +the vendored version of osm2pgsql, you need to set the PATH accordingly. + ### Installing prerequisites on Ubuntu/Debian The Python tools should always be run with the most recent version. diff --git a/docs/develop/Testing.md b/docs/develop/Testing.md index 97d40ab7..aeaca2ee 100644 --- a/docs/develop/Testing.md +++ b/docs/develop/Testing.md @@ -78,7 +78,6 @@ To run the functional tests, do The tests can be configured with a set of environment variables (`behave -D key=val`): - * `BUILDDIR` - build directory of Nominatim installation to test * `TEMPLATE_DB` - name of template database used as a skeleton for the test databases (db tests) * `TEST_DB` - name of test database (db tests) @@ -91,7 +90,7 @@ The tests can be configured with a set of environment variables (`behave -D key= * `DB_USER` - (optional) username of database login * `DB_PASS` - (optional) password for database login * `SERVER_MODULE_PATH` - (optional) path on the Postgres server to Nominatim - module shared library file + module shared library file (only needed for legacy tokenizer) * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused during the next run. Reusing the base templates speeds up tests considerably but might lead to outdated errors diff --git a/test/bdd/environment.py b/test/bdd/environment.py index 85896898..6aec8f59 100644 --- a/test/bdd/environment.py +++ b/test/bdd/environment.py @@ -5,16 +5,18 @@ # Copyright (C) 2024 by the Nominatim developer community. # For a full list of authors see the git log. from pathlib import Path +import sys from behave import * +sys.path.insert(1, str(Path(__file__, '..', '..', '..', 'src').resolve())) + from steps.geometry_factory import GeometryFactory from steps.nominatim_environment import NominatimEnvironment -TEST_BASE_DIR = Path(__file__) / '..' / '..' +TEST_BASE_DIR = Path(__file__, '..', '..').resolve() userconfig = { - 'BUILDDIR' : (TEST_BASE_DIR / '..' / 'build').resolve(), 'REMOVE_TEMPLATE' : False, 'KEEP_TEST_DB' : False, 'DB_HOST' : None, @@ -24,7 +26,7 @@ userconfig = { 'TEMPLATE_DB' : 'test_template_nominatim', 'TEST_DB' : 'test_nominatim', 'API_TEST_DB' : 'test_api_nominatim', - 'API_TEST_FILE' : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(), + 'API_TEST_FILE' : TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf', 'SERVER_MODULE_PATH' : None, 'TOKENIZER' : None, # Test with a custom tokenizer 'STYLE' : 'extratags', diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 17a76745..88a4f11c 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -6,14 +6,11 @@ # For a full list of authors see the git log. from pathlib import Path import importlib -import sys import tempfile import psycopg from psycopg import sql as pysql -sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..'/ 'src').resolve())) - from nominatim_db import cli from nominatim_db.config import Configuration from nominatim_db.db.connection import Connection, register_hstore, execute_scalar @@ -26,7 +23,6 @@ class NominatimEnvironment: """ def __init__(self, config): - self.build_dir = Path(config['BUILDDIR']).resolve() self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve() self.db_host = config['DB_HOST'] self.db_port = config['DB_PORT'] @@ -56,6 +52,9 @@ class NominatimEnvironment: raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'") self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")() + if self.tokenizer == 'legacy' and self.server_module_path is None: + raise RuntimeError("You must set -DSERVER_MODULE_PATH when testing the legacy tokenizer.") + def connect_database(self, dbname): """ Return a connection to the database with the given name. Uses configured host, user and port. @@ -107,8 +106,6 @@ class NominatimEnvironment: self.test_env['NOMINATIM_DATADIR'] = str((self.src_dir / 'data').resolve()) self.test_env['NOMINATIM_SQLDIR'] = str((self.src_dir / 'lib-sql').resolve()) self.test_env['NOMINATIM_CONFIGDIR'] = str((self.src_dir / 'settings').resolve()) - self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str((self.build_dir / 'module').resolve()) - self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = str((self.build_dir / 'osm2pgsql' / 'osm2pgsql').resolve()) if self.tokenizer is not None: self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer if self.import_style is not None: @@ -116,9 +113,6 @@ class NominatimEnvironment: if self.server_module_path: self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path - else: - # avoid module being copied into the temporary environment - self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = str((self.build_dir / 'module').resolve()) if self.website_dir is not None: self.website_dir.cleanup() @@ -137,8 +131,7 @@ class NominatimEnvironment: def get_test_config(self): cfg = Configuration(Path(self.website_dir.name), environ=self.test_env) - cfg.set_libdirs(module=self.build_dir / 'module', - osm2pgsql=self.build_dir / 'osm2pgsql' / 'osm2pgsql') + cfg.set_libdirs(module=self.server_module_path) return cfg def get_libpq_dsn(self): @@ -305,8 +298,8 @@ class NominatimEnvironment: if self.website_dir is not None: cmdline = list(cmdline) + ['--project-dir', self.website_dir.name] - cli.nominatim(module_dir='', - osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'), + cli.nominatim(module_dir=self.server_module_path, + osm2pgsql_path=None, cli_args=cmdline, environ=self.test_env) diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py index 0c1b421d..4cee75f7 100644 --- a/test/bdd/steps/steps_osm_data.py +++ b/test/bdd/steps/steps_osm_data.py @@ -16,7 +16,7 @@ from geometry_alias import ALIASES def get_osm2pgsql_options(nominatim_env, fname, append): return dict(import_file=fname, - osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'), + osm2pgsql='osm2pgsql', osm2pgsql_cache=50, osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()), osm2pgsql_style_path=nominatim_env.get_test_config().config_dir, -- 2.39.5