bdd tests: get rid of dependency on build path

BDD tests will now use whatever osm2pgsql they find in the PATH.
For testing against legacy tokenizer, use -DSERVER_MODULE_PATH
to point to the module. This will no longer work out of the box.

diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md
index a7c474e85bbc4248e44fb670317f3d3b657f3dcf..19948e7cc229bde18ce159462ec18e3211255fb3 100644 (file)
--- a/docs/develop/Development-Environment.md
+++ b/docs/develop/Development-Environment.md
@@ -50,6 +50,10 @@ The documentation is built with mkdocs:
 * [mkdocstrings](https://mkdocstrings.github.io/) >= 0.25
 * [mkdocs-material](https://squidfunk.github.io/mkdocs-material/)
 
 * [mkdocstrings](https://mkdocstrings.github.io/) >= 0.25
 * [mkdocs-material](https://squidfunk.github.io/mkdocs-material/)
 

+Please be aware that tests always run against the globally installed
+osm2pgsql, so you need to have this set up. If you want to test against
+the vendored version of osm2pgsql, you need to set the PATH accordingly.
+

 ### Installing prerequisites on Ubuntu/Debian
 
 The Python tools should always be run with the most recent version.
 ### Installing prerequisites on Ubuntu/Debian
 
 The Python tools should always be run with the most recent version.

diff --git a/docs/develop/Testing.md b/docs/develop/Testing.md
index 97d40ab7375a2d6a426e74896c886608c5ab4ca2..aeaca2ee818a84addcae57dc60e9388f03277c11 100644 (file)
--- a/docs/develop/Testing.md
+++ b/docs/develop/Testing.md
@@ -78,7 +78,6 @@ To run the functional tests, do
 
 The tests can be configured with a set of environment variables (`behave -D key=val`):
 
 
 The tests can be configured with a set of environment variables (`behave -D key=val`):
 

- * `BUILDDIR` - build directory of Nominatim installation to test

  * `TEMPLATE_DB` - name of template database used as a skeleton for
                    the test databases (db tests)
  * `TEST_DB` - name of test database (db tests)
  * `TEMPLATE_DB` - name of template database used as a skeleton for
                    the test databases (db tests)
  * `TEST_DB` - name of test database (db tests)


@@ -91,7 +90,7 @@ The tests can be configured with a set of environment variables (`behave -D key=
  * `DB_USER` - (optional) username of database login
  * `DB_PASS` - (optional) password for database login
  * `SERVER_MODULE_PATH` - (optional) path on the Postgres server to Nominatim
  * `DB_USER` - (optional) username of database login
  * `DB_PASS` - (optional) password for database login
  * `SERVER_MODULE_PATH` - (optional) path on the Postgres server to Nominatim

-                          module shared library file
+                          module shared library file (only needed for legacy tokenizer)

  * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused
                        during the next run. Reusing the base templates speeds
                        up tests considerably but might lead to outdated errors
  * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused
                        during the next run. Reusing the base templates speeds
                        up tests considerably but might lead to outdated errors

diff --git a/test/bdd/environment.py b/test/bdd/environment.py
index 8589689852852b9c2450bef8ab4a645b7b6ffdb5..6aec8f597a39dcf5676197fb3288589ea1118561 100644 (file)
--- a/test/bdd/environment.py
+++ b/test/bdd/environment.py
@@ -5,16 +5,18 @@
 # Copyright (C) 2024 by the Nominatim developer community.
 # For a full list of authors see the git log.
 from pathlib import Path
 # Copyright (C) 2024 by the Nominatim developer community.
 # For a full list of authors see the git log.
 from pathlib import Path

+import sys

 
 from behave import *
 
 
 from behave import *
 

+sys.path.insert(1, str(Path(__file__, '..', '..', '..', 'src').resolve()))
+

 from steps.geometry_factory import GeometryFactory
 from steps.nominatim_environment import NominatimEnvironment
 
 from steps.geometry_factory import GeometryFactory
 from steps.nominatim_environment import NominatimEnvironment
 

-TEST_BASE_DIR = Path(__file__) / '..' / '..'
+TEST_BASE_DIR = Path(__file__, '..', '..').resolve()

 
 userconfig = {
 
 userconfig = {

-    'BUILDDIR' : (TEST_BASE_DIR / '..' / 'build').resolve(),

     'REMOVE_TEMPLATE' : False,
     'KEEP_TEST_DB' : False,
     'DB_HOST' : None,
     'REMOVE_TEMPLATE' : False,
     'KEEP_TEST_DB' : False,
     'DB_HOST' : None,


@@ -24,7 +26,7 @@ userconfig = {
     'TEMPLATE_DB' : 'test_template_nominatim',
     'TEST_DB' : 'test_nominatim',
     'API_TEST_DB' : 'test_api_nominatim',
     'TEMPLATE_DB' : 'test_template_nominatim',
     'TEST_DB' : 'test_nominatim',
     'API_TEST_DB' : 'test_api_nominatim',

-    'API_TEST_FILE'  : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(),
+    'API_TEST_FILE'  : TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf',

     'SERVER_MODULE_PATH' : None,
     'TOKENIZER' : None, # Test with a custom tokenizer
     'STYLE' : 'extratags',
     'SERVER_MODULE_PATH' : None,
     'TOKENIZER' : None, # Test with a custom tokenizer
     'STYLE' : 'extratags',

diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py
index 17a7674507b844ae1983524aa04e8d772cf849fb..88a4f11c73ebc5ec135b0443660cce3f14753731 100644 (file)
--- a/test/bdd/steps/nominatim_environment.py
+++ b/test/bdd/steps/nominatim_environment.py
@@ -6,14 +6,11 @@
 # For a full list of authors see the git log.
 from pathlib import Path
 import importlib
 # For a full list of authors see the git log.
 from pathlib import Path
 import importlib

-import sys

 import tempfile
 
 import psycopg
 from psycopg import sql as pysql
 
 import tempfile
 
 import psycopg
 from psycopg import sql as pysql
 

-sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..'/ 'src').resolve()))
-

 from nominatim_db import cli
 from nominatim_db.config import Configuration
 from nominatim_db.db.connection import Connection, register_hstore, execute_scalar
 from nominatim_db import cli
 from nominatim_db.config import Configuration
 from nominatim_db.db.connection import Connection, register_hstore, execute_scalar


@@ -26,7 +23,6 @@ class NominatimEnvironment:
     """
 
     def __init__(self, config):
     """
 
     def __init__(self, config):

-        self.build_dir = Path(config['BUILDDIR']).resolve()

         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
         self.db_host = config['DB_HOST']
         self.db_port = config['DB_PORT']
         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
         self.db_host = config['DB_HOST']
         self.db_port = config['DB_PORT']


@@ -56,6 +52,9 @@ class NominatimEnvironment:
                 raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'")
             self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")()
 
                 raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'")
             self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")()
 

+        if self.tokenizer == 'legacy' and self.server_module_path is None:
+            raise RuntimeError("You must set -DSERVER_MODULE_PATH when testing the legacy tokenizer.")
+

     def connect_database(self, dbname):
         """ Return a connection to the database with the given name.
             Uses configured host, user and port.
     def connect_database(self, dbname):
         """ Return a connection to the database with the given name.
             Uses configured host, user and port.


@@ -107,8 +106,6 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_DATADIR'] = str((self.src_dir / 'data').resolve())
         self.test_env['NOMINATIM_SQLDIR'] = str((self.src_dir / 'lib-sql').resolve())
         self.test_env['NOMINATIM_CONFIGDIR'] = str((self.src_dir / 'settings').resolve())
         self.test_env['NOMINATIM_DATADIR'] = str((self.src_dir / 'data').resolve())
         self.test_env['NOMINATIM_SQLDIR'] = str((self.src_dir / 'lib-sql').resolve())
         self.test_env['NOMINATIM_CONFIGDIR'] = str((self.src_dir / 'settings').resolve())

-        self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str((self.build_dir / 'module').resolve())
-        self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = str((self.build_dir / 'osm2pgsql' / 'osm2pgsql').resolve())

         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
         if self.import_style is not None:
         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
         if self.import_style is not None:


@@ -116,9 +113,6 @@ class NominatimEnvironment:
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path

-        else:
-            # avoid module being copied into the temporary environment
-            self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = str((self.build_dir / 'module').resolve())

 
         if self.website_dir is not None:
             self.website_dir.cleanup()
 
         if self.website_dir is not None:
             self.website_dir.cleanup()


@@ -137,8 +131,7 @@ class NominatimEnvironment:
 
     def get_test_config(self):
         cfg = Configuration(Path(self.website_dir.name), environ=self.test_env)
 
     def get_test_config(self):
         cfg = Configuration(Path(self.website_dir.name), environ=self.test_env)

-        cfg.set_libdirs(module=self.build_dir / 'module',
-                        osm2pgsql=self.build_dir / 'osm2pgsql' / 'osm2pgsql')
+        cfg.set_libdirs(module=self.server_module_path)

         return cfg
 
     def get_libpq_dsn(self):
         return cfg
 
     def get_libpq_dsn(self):


@@ -305,8 +298,8 @@ class NominatimEnvironment:
         if self.website_dir is not None:
             cmdline = list(cmdline) + ['--project-dir', self.website_dir.name]
 
         if self.website_dir is not None:
             cmdline = list(cmdline) + ['--project-dir', self.website_dir.name]
 

-        cli.nominatim(module_dir='',
-                      osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'),
+        cli.nominatim(module_dir=self.server_module_path,
+                      osm2pgsql_path=None,

                       cli_args=cmdline,
                       environ=self.test_env)
 
                       cli_args=cmdline,
                       environ=self.test_env)
 

diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py
index 0c1b421d950b0a45960fb9bb29c2f0a6717c541b..4cee75f7a32186b47ac8604a9d1a7969f0f6d6a4 100644 (file)
--- a/test/bdd/steps/steps_osm_data.py
+++ b/test/bdd/steps/steps_osm_data.py
@@ -16,7 +16,7 @@ from geometry_alias import ALIASES
 
 def get_osm2pgsql_options(nominatim_env, fname, append):
     return dict(import_file=fname,
 
 def get_osm2pgsql_options(nominatim_env, fname, append):
     return dict(import_file=fname,

-                osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
+                osm2pgsql='osm2pgsql',

                 osm2pgsql_cache=50,
                 osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
                 osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,
                 osm2pgsql_cache=50,
                 osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
                 osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,