From 2abe9e6fd99bb270c9de05f19701283a08c3b2f5 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 26 Nov 2022 22:00:43 +0100 Subject: [PATCH] use data paths from new nominatim.paths --- cmake/tool-installed.tmpl | 4 -- cmake/tool.tmpl | 4 -- nominatim/cli.py | 15 +++----- nominatim/clicmd/admin.py | 2 +- nominatim/clicmd/args.py | 8 +--- nominatim/clicmd/setup.py | 2 +- nominatim/config.py | 25 +++++++------ nominatim/tokenizer/factory.py | 2 + nominatim/tokenizer/legacy_tokenizer.py | 7 ++++ nominatim/tools/exec_utils.py | 19 +++++----- nominatim/tools/refresh.py | 1 + test/bdd/steps/nominatim_environment.py | 10 ++--- test/python/cli/conftest.py | 4 -- test/python/cli/test_cli.py | 8 ++-- test/python/cli/test_cmd_replication.py | 3 +- test/python/config/test_config.py | 8 ++-- test/python/config/test_config_load_module.py | 2 +- test/python/conftest.py | 23 ++++-------- test/python/tools/test_exec_utils.py | 37 +++++++++---------- 19 files changed, 80 insertions(+), 104 deletions(-) diff --git a/cmake/tool-installed.tmpl b/cmake/tool-installed.tmpl index d1c453fa..e38dafab 100644 --- a/cmake/tool-installed.tmpl +++ b/cmake/tool-installed.tmpl @@ -11,8 +11,4 @@ version.GIT_COMMIT_HASH = '@GIT_HASH@' exit(cli.nominatim(module_dir='@NOMINATIM_LIBDIR@/module', osm2pgsql_path='@NOMINATIM_LIBDIR@/osm2pgsql', - phplib_dir='@NOMINATIM_LIBDIR@/lib-php', - sqllib_dir='@NOMINATIM_LIBDIR@/lib-sql', - data_dir='@NOMINATIM_DATADIR@', - config_dir='@NOMINATIM_CONFIGDIR@', phpcgi_path='@PHPCGI_BIN@')) diff --git a/cmake/tool.tmpl b/cmake/tool.tmpl index b76d2e0a..96c6c6dc 100755 --- a/cmake/tool.tmpl +++ b/cmake/tool.tmpl @@ -11,8 +11,4 @@ version.GIT_COMMIT_HASH = '@GIT_HASH@' exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module', osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql', - phplib_dir='@CMAKE_SOURCE_DIR@/lib-php', - sqllib_dir='@CMAKE_SOURCE_DIR@/lib-sql', - data_dir='@CMAKE_SOURCE_DIR@/data', - config_dir='@CMAKE_SOURCE_DIR@/settings', phpcgi_path='@PHPCGI_BIN@')) diff --git a/nominatim/cli.py b/nominatim/cli.py index 8c2136f4..56ed6a07 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -100,9 +100,7 @@ class CommandlineParser: self.parser.print_help() return 1 - for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'sqllib_dir', - 'data_dir', 'config_dir', 'phpcgi_path'): - setattr(args, arg, Path(kwargs[arg])) + args.phpcgi_path = Path(kwargs['phpcgi_path']) args.project_dir = Path(args.project_dir).resolve() if 'cli_args' not in kwargs: @@ -111,13 +109,10 @@ class CommandlineParser: datefmt='%Y-%m-%d %H:%M:%S', level=max(4 - args.verbose, 1) * 10) - args.config = Configuration(args.project_dir, args.config_dir, + args.config = Configuration(args.project_dir, environ=kwargs.get('environ', os.environ)) - args.config.set_libdirs(module=args.module_dir, - osm2pgsql=args.osm2pgsql_path, - php=args.phplib_dir, - sql=args.sqllib_dir, - data=args.data_dir) + args.config.set_libdirs(module=kwargs['module_dir'], + osm2pgsql=kwargs['osm2pgsql_path']) log = logging.getLogger() log.warning('Using project directory: %s', str(args.project_dir)) @@ -195,7 +190,7 @@ class QueryExport: if args.restrict_to_osm_relation: params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation)) - return run_legacy_script('export.php', *params, nominatim_env=args) + return run_legacy_script('export.php', *params, config=args.config) class AdminServe: diff --git a/nominatim/clicmd/admin.py b/nominatim/clicmd/admin.py index c3ba9c0b..0c773960 100644 --- a/nominatim/clicmd/admin.py +++ b/nominatim/clicmd/admin.py @@ -88,4 +88,4 @@ class AdminFuncs: params.append('--reverse-only') if args.target == 'search': params.append('--search-only') - return run_legacy_script(*params, nominatim_env=args) + return run_legacy_script(*params, config=args.config) diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py index 45599ad5..98673ed6 100644 --- a/nominatim/clicmd/args.py +++ b/nominatim/clicmd/args.py @@ -42,12 +42,6 @@ class NominatimArgs: # Basic environment set by root program. config: Configuration project_dir: Path - module_dir: Path - osm2pgsql_path: Path - phplib_dir: Path - sqllib_dir: Path - data_dir: Path - config_dir: Path phpcgi_path: Path # Global switches @@ -181,7 +175,7 @@ class NominatimArgs: from the command line arguments. The resulting dict can be further customized and then used in `run_osm2pgsql()`. """ - return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path, + return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.config.lib_dir.osm2pgsql, osm2pgsql_cache=self.osm2pgsql_cache or default_cache, osm2pgsql_style=self.config.get_import_style_file(), osm2pgsql_style_path=self.config.config_dir, diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index 344167bb..68884fe1 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -87,7 +87,7 @@ class SetupAll: LOG.warning('Setting up country tables') country_info.setup_country_tables(args.config.get_libpq_dsn(), - args.data_dir, + args.config.lib_dir.data, args.no_partitions) LOG.warning('Importing OSM data file') diff --git a/nominatim/config.py b/nominatim/config.py index 7502aff7..1728c291 100644 --- a/nominatim/config.py +++ b/nominatim/config.py @@ -20,6 +20,7 @@ from dotenv import dotenv_values from nominatim.typing import StrPath from nominatim.errors import UsageError +import nominatim.paths LOG = logging.getLogger() CONFIG_CACHE : Dict[str, Any] = {} @@ -58,21 +59,22 @@ class Configuration: avoid conflicts with other environment variables. """ - def __init__(self, project_dir: Path, config_dir: Path, + def __init__(self, project_dir: Optional[Path], environ: Optional[Mapping[str, str]] = None) -> None: self.environ = environ or os.environ self.project_dir = project_dir - self.config_dir = config_dir - self._config = dotenv_values(str((config_dir / 'env.defaults').resolve())) - if project_dir is not None and (project_dir / '.env').is_file(): - self._config.update(dotenv_values(str((project_dir / '.env').resolve()))) + self.config_dir = nominatim.paths.CONFIG_DIR + self._config = dotenv_values(str(self.config_dir / 'env.defaults')) + if self.project_dir is not None and (self.project_dir / '.env').is_file(): + self.project_dir = self.project_dir.resolve() + self._config.update(dotenv_values(str(self.project_dir / '.env'))) class _LibDirs: module: Path osm2pgsql: Path - php: Path - sql: Path - data: Path + php = nominatim.paths.PHPLIB_DIR + sql = nominatim.paths.SQLLIB_DIR + data = nominatim.paths.DATA_DIR self.lib_dir = _LibDirs() self._private_plugins: Dict[str, object] = {} @@ -82,7 +84,7 @@ class Configuration: """ Set paths to library functions and data. """ for key, value in kwargs.items(): - setattr(self.lib_dir, key, Path(value).resolve()) + setattr(self.lib_dir, key, Path(value)) def __getattr__(self, name: str) -> str: @@ -136,6 +138,7 @@ class Configuration: cfgpath = Path(value) if not cfgpath.is_absolute(): + assert self.project_dir is not None cfgpath = self.project_dir / cfgpath return cfgpath.resolve() @@ -174,11 +177,11 @@ class Configuration: return self.find_config_file('', 'IMPORT_STYLE') - def get_os_env(self) -> Dict[str, Optional[str]]: + def get_os_env(self) -> Dict[str, str]: """ Return a copy of the OS environment with the Nominatim configuration merged in. """ - env = dict(self._config) + env = {k: v for k, v in self._config.items() if v is not None} env.update(self.environ) return env diff --git a/nominatim/tokenizer/factory.py b/nominatim/tokenizer/factory.py index f5159fa0..d6bc5163 100644 --- a/nominatim/tokenizer/factory.py +++ b/nominatim/tokenizer/factory.py @@ -55,6 +55,7 @@ def create_tokenizer(config: Configuration, init_db: bool = True, module_name = config.TOKENIZER # Create the directory for the tokenizer data + assert config.project_dir is not None basedir = config.project_dir / 'tokenizer' if not basedir.exists(): basedir.mkdir() @@ -80,6 +81,7 @@ def get_tokenizer_for_db(config: Configuration) -> AbstractTokenizer: The function looks up the appropriate tokenizer in the database and initialises it. """ + assert config.project_dir is not None basedir = config.project_dir / 'tokenizer' if not basedir.is_dir(): # Directory will be repopulated by tokenizer below. diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py index 1b0b2980..a50dedb2 100644 --- a/nominatim/tokenizer/legacy_tokenizer.py +++ b/nominatim/tokenizer/legacy_tokenizer.py @@ -106,6 +106,7 @@ class LegacyTokenizer(AbstractTokenizer): This copies all necessary data in the project directory to make sure the tokenizer remains stable even over updates. """ + assert config.project_dir is not None module_dir = _install_module(config.DATABASE_MODULE_PATH, config.lib_dir.module, config.project_dir / 'module') @@ -127,6 +128,8 @@ class LegacyTokenizer(AbstractTokenizer): def init_from_project(self, config: Configuration) -> None: """ Initialise the tokenizer from the project directory. """ + assert config.project_dir is not None + with connect(self.dsn) as conn: self.normalization = properties.get_property(conn, DBCFG_NORMALIZATION) @@ -149,6 +152,8 @@ class LegacyTokenizer(AbstractTokenizer): def update_sql_functions(self, config: Configuration) -> None: """ Reimport the SQL functions for this tokenizer. """ + assert config.project_dir is not None + with connect(self.dsn) as conn: max_word_freq = properties.get_property(conn, DBCFG_MAXWORDFREQ) modulepath = config.DATABASE_MODULE_PATH or \ @@ -193,6 +198,8 @@ class LegacyTokenizer(AbstractTokenizer): This is a special migration function for updating existing databases to new software versions. """ + assert config.project_dir is not None + self.normalization = config.TERM_NORMALIZATION module_dir = _install_module(config.DATABASE_MODULE_PATH, config.lib_dir.module, diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index ed3bb53b..9c801142 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -15,6 +15,7 @@ import subprocess import urllib.request as urlrequest from urllib.parse import urlencode +from nominatim.config import Configuration from nominatim.typing import StrPath from nominatim.version import version_str from nominatim.db.connection import get_pg_env @@ -22,7 +23,7 @@ from nominatim.db.connection import get_pg_env LOG = logging.getLogger() def run_legacy_script(script: StrPath, *args: Union[int, str], - nominatim_env: Any, + config: Configuration, throw_on_fail: bool = False) -> int: """ Run a Nominatim PHP script with the given arguments. @@ -30,18 +31,18 @@ def run_legacy_script(script: StrPath, *args: Union[int, str], then throw a `CalledProcessError` on a non-zero exit. """ cmd = ['/usr/bin/env', 'php', '-Cq', - str(nominatim_env.phplib_dir / 'admin' / script)] + str(config.lib_dir.php / 'admin' / script)] cmd.extend([str(a) for a in args]) - env = nominatim_env.config.get_os_env() - env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir) - env['NOMINATIM_SQLDIR'] = str(nominatim_env.sqllib_dir) - env['NOMINATIM_CONFIGDIR'] = str(nominatim_env.config_dir) - env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str(nominatim_env.module_dir) + env = config.get_os_env() + env['NOMINATIM_DATADIR'] = str(config.lib_dir.data) + env['NOMINATIM_SQLDIR'] = str(config.lib_dir.sql) + env['NOMINATIM_CONFIGDIR'] = str(config.config_dir) + env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str(config.lib_dir.module) if not env['NOMINATIM_OSM2PGSQL_BINARY']: - env['NOMINATIM_OSM2PGSQL_BINARY'] = str(nominatim_env.osm2pgsql_path) + env['NOMINATIM_OSM2PGSQL_BINARY'] = str(config.lib_dir.osm2pgsql) - proc = subprocess.run(cmd, cwd=str(nominatim_env.project_dir), env=env, + proc = subprocess.run(cmd, cwd=str(config.project_dir), env=env, check=throw_on_fail) return proc.returncode diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index c50493cc..b35d3aae 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -216,6 +216,7 @@ def setup_website(basedir: Path, config: Configuration, conn: Connection) -> Non LOG.info('Creating website directory.') basedir.mkdir() + assert config.project_dir is not None template = dedent(f"""\ = 0 ? 0 : 10); """) + monkeypatch.setattr(nominatim.paths, 'PHPLIB_DIR', tmp_path) + assert nominatim.cli.nominatim(module_dir='MODULE NOT AVAILABLE', osm2pgsql_path='OSM2PGSQL NOT AVAILABLE', - phplib_dir=str(tmp_path), - data_dir=str(src_dir / 'data'), phpcgi_path='/usr/bin/php-cgi', - sqllib_dir=str(src_dir / 'lib-sql'), - config_dir=str(src_dir / 'settings'), cli_args=['export', '--' + param, value]) == 0 diff --git a/test/python/cli/test_cmd_replication.py b/test/python/cli/test_cmd_replication.py index 9fd4f5b3..a22d077a 100644 --- a/test/python/cli/test_cmd_replication.py +++ b/test/python/cli/test_cmd_replication.py @@ -15,6 +15,7 @@ import pytest import nominatim.cli import nominatim.indexer.indexer import nominatim.tools.replication +import nominatim.tools.refresh from nominatim.db import status @pytest.fixture @@ -107,7 +108,7 @@ class TestCliReplication: def test_replication_update_once_no_index(self, update_mock): assert self.call_nominatim('--once', '--no-index') == 0 - assert str(update_mock.last_args[1]['osm2pgsql']) == 'OSM2PGSQL NOT AVAILABLE' + assert str(update_mock.last_args[1]['osm2pgsql']).endswith('OSM2PGSQL NOT AVAILABLE') def test_replication_update_custom_osm2pgsql(self, monkeypatch, update_mock): diff --git a/test/python/config/test_config.py b/test/python/config/test_config.py index a9cbb48d..a003065d 100644 --- a/test/python/config/test_config.py +++ b/test/python/config/test_config.py @@ -14,23 +14,23 @@ from nominatim.config import Configuration, flatten_config_list from nominatim.errors import UsageError @pytest.fixture -def make_config(src_dir): +def make_config(): """ Create a configuration object from the given project directory. """ def _mk_config(project_dir=None): - return Configuration(project_dir, src_dir / 'settings') + return Configuration(project_dir) return _mk_config @pytest.fixture -def make_config_path(src_dir, tmp_path): +def make_config_path(tmp_path): """ Create a configuration object with project and config directories in a temporary directory. """ def _mk_config(): (tmp_path / 'project').mkdir() (tmp_path / 'config').mkdir() - conf = Configuration(tmp_path / 'project', src_dir / 'settings') + conf = Configuration(tmp_path / 'project') conf.config_dir = tmp_path / 'config' return conf diff --git a/test/python/config/test_config_load_module.py b/test/python/config/test_config_load_module.py index df6c4794..7bc91fd7 100644 --- a/test/python/config/test_config_load_module.py +++ b/test/python/config/test_config_load_module.py @@ -21,7 +21,7 @@ def test_config(src_dir, tmp_path): """ (tmp_path / 'project').mkdir() (tmp_path / 'config').mkdir() - conf = Configuration(tmp_path / 'project', src_dir / 'settings') + conf = Configuration(tmp_path / 'project') conf.config_dir = tmp_path / 'config' return conf diff --git a/test/python/conftest.py b/test/python/conftest.py index 40526295..31463746 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -107,24 +107,18 @@ def table_factory(temp_db_cursor): @pytest.fixture -def def_config(src_dir): - cfg = Configuration(None, src_dir / 'settings') - cfg.set_libdirs(module='.', osm2pgsql='.', - php=src_dir / 'lib-php', - sql=src_dir / 'lib-sql', - data=src_dir / 'data') +def def_config(): + cfg = Configuration(None) + cfg.set_libdirs(module='.', osm2pgsql='.') return cfg @pytest.fixture -def project_env(src_dir, tmp_path): +def project_env(tmp_path): projdir = tmp_path / 'project' projdir.mkdir() - cfg = Configuration(projdir, src_dir / 'settings') - cfg.set_libdirs(module='.', osm2pgsql='.', - php=src_dir / 'lib-php', - sql=src_dir / 'lib-sql', - data=src_dir / 'data') + cfg = Configuration(projdir) + cfg.set_libdirs(module='.', osm2pgsql='.') return cfg @@ -214,9 +208,8 @@ def osmline_table(temp_db_with_extensions, table_factory): @pytest.fixture def sql_preprocessor_cfg(tmp_path, table_factory, temp_db_with_extensions): table_factory('country_name', 'partition INT', ((0, ), (1, ), (2, ))) - cfg = Configuration(None, SRC_DIR.resolve() / 'settings') - cfg.set_libdirs(module='.', osm2pgsql='.', php=SRC_DIR / 'lib-php', - sql=tmp_path, data=SRC_DIR / 'data') + cfg = Configuration(None) + cfg.set_libdirs(module='.', osm2pgsql='.', sql=tmp_path) return cfg diff --git a/test/python/tools/test_exec_utils.py b/test/python/tools/test_exec_utils.py index 26ea92b2..f73aec30 100644 --- a/test/python/tools/test_exec_utils.py +++ b/test/python/tools/test_exec_utils.py @@ -12,31 +12,28 @@ import subprocess import pytest +from nominatim.config import Configuration import nominatim.tools.exec_utils as exec_utils +import nominatim.paths class TestRunLegacyScript: @pytest.fixture(autouse=True) - def setup_nominatim_env(self, tmp_path, def_config): + def setup_nominatim_env(self, tmp_path, monkeypatch): tmp_phplib_dir = tmp_path / 'phplib' tmp_phplib_dir.mkdir() (tmp_phplib_dir / 'admin').mkdir() - class _NominatimEnv: - config = def_config - phplib_dir = tmp_phplib_dir - data_dir = Path('data') - project_dir = Path('.') - sqllib_dir = Path('lib-sql') - config_dir = Path('settings') - module_dir = 'module' - osm2pgsql_path = 'osm2pgsql' + monkeypatch.setattr(nominatim.paths, 'PHPLIB_DIR', tmp_phplib_dir) - self.testenv = _NominatimEnv + self.phplib_dir = tmp_phplib_dir + self.config = Configuration(tmp_path) + self.config.set_libdirs(module='.', osm2pgsql='default_osm2pgsql', + php=tmp_phplib_dir) def mk_script(self, code): - codefile = self.testenv.phplib_dir / 'admin' / 't.php' + codefile = self.phplib_dir / 'admin' / 't.php' codefile.write_text('