From: Sarah Hoffmann Date: Wed, 24 Feb 2021 09:38:19 +0000 (+0100) Subject: introduce custom object for cmdline arguments X-Git-Tag: v3.7.0~29^2~13 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/7222235579bc0d287c86fea6d7e22959132c7377 introduce custom object for cmdline arguments Allows to define special functions over the arguments. Also splits CLI tests in two files as they have become too many. --- diff --git a/nominatim/cli.py b/nominatim/cli.py index 8668c51c..e1824cc6 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -12,6 +12,7 @@ from .config import Configuration from .tools.exec_utils import run_legacy_script, run_php_server from .errors import UsageError from . import clicmd +from .clicmd.args import NominatimArgs LOG = logging.getLogger() @@ -62,7 +63,8 @@ class CommandlineParser: """ Parse the command line arguments of the program and execute the appropriate subcommand. """ - args = self.parser.parse_args(args=kwargs.get('cli_args')) + args = NominatimArgs() + self.parser.parse_args(args=kwargs.get('cli_args'), namespace=args) if args.subcommand is None: self.parser.print_help() diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py new file mode 100644 index 00000000..f9b94ef2 --- /dev/null +++ b/nominatim/clicmd/args.py @@ -0,0 +1,22 @@ +""" +Provides custom functions over command-line arguments. +""" + + +class NominatimArgs: + """ Customized namespace class for the nominatim command line tool + to receive the command-line arguments. + """ + + def osm2pgsql_options(self, default_cache, default_threads): + """ Return the standard osm2pgsql options that can be derived + from the command line arguments. The resulting dict can be + further customized and then used in `run_osm2pgsql()`. + """ + return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path, + osm2pgsql_cache=self.osm2pgsql_cache or default_cache, + osm2pgsql_style=self.config.get_import_style_file(), + threads=self.threads or default_threads, + dsn=self.config.get_libpq_dsn(), + flatnode_file=self.config.FLATNODE_FILE) + diff --git a/nominatim/clicmd/replication.py b/nominatim/clicmd/replication.py index e766be2b..fc18945e 100644 --- a/nominatim/clicmd/replication.py +++ b/nominatim/clicmd/replication.py @@ -17,17 +17,6 @@ LOG = logging.getLogger() # Using non-top-level imports to make pyosmium optional for replication only. # pylint: disable=E0012,C0415 -def _osm2pgsql_options_from_args(args, default_cache, default_threads): - """ Set up the standard osm2pgsql from the command line arguments. - """ - return dict(osm2pgsql=args.osm2pgsql_path, - osm2pgsql_cache=args.osm2pgsql_cache or default_cache, - osm2pgsql_style=args.config.get_import_style_file(), - threads=args.threads or default_threads, - dsn=args.config.get_libpq_dsn(), - flatnode_file=args.config.FLATNODE_FILE) - - class UpdateReplication: """\ Update the database using an online replication service. @@ -96,7 +85,7 @@ class UpdateReplication: from ..tools import replication from ..indexer.indexer import Indexer - params = _osm2pgsql_options_from_args(args, 2000, 1) + params = args.osm2pgsql_options(default_cache=2000, default_threads=1) params.update(base_url=args.config.REPLICATION_URL, update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'), import_file=args.project_dir / 'osmosischange.osc', diff --git a/test/python/mocks.py b/test/python/mocks.py new file mode 100644 index 00000000..415e18b3 --- /dev/null +++ b/test/python/mocks.py @@ -0,0 +1,18 @@ +""" +Custom mocks for testing. +""" + + +class MockParamCapture: + """ Mock that records the parameters with which a function was called + as well as the number of calls. + """ + def __init__(self, retval=0): + self.called = 0 + self.return_value = retval + + def __call__(self, *args, **kwargs): + self.called += 1 + self.last_args = args + self.last_kwargs = kwargs + return self.return_value diff --git a/test/python/test_cli.py b/test/python/test_cli.py index 03325b8d..2b4240b4 100644 --- a/test/python/test_cli.py +++ b/test/python/test_cli.py @@ -5,11 +5,8 @@ These tests just check that the various command line parameters route to the correct functionionality. They use a lot of monkeypatching to avoid executing the actual functions. """ -import datetime as dt -import time from pathlib import Path -import psycopg2 import pytest import nominatim.cli @@ -21,9 +18,8 @@ import nominatim.tools.admin import nominatim.tools.check_database import nominatim.tools.freeze import nominatim.tools.refresh -import nominatim.tools.replication -from nominatim.errors import UsageError -from nominatim.db import status + +from mocks import MockParamCapture SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve() @@ -37,19 +33,6 @@ def call_nominatim(*args): config_dir=str(SRC_DIR / 'settings'), cli_args=args) -class MockParamCapture: - """ Mock that records the parameters with which a function was called - as well as the number of calls. - """ - def __init__(self, retval=0): - self.called = 0 - self.return_value = retval - - def __call__(self, *args, **kwargs): - self.called += 1 - self.last_args = args - self.last_kwargs = kwargs - return self.return_value @pytest.fixture def mock_run_legacy(monkeypatch): @@ -186,79 +169,6 @@ def test_refresh_importance_computed_after_wiki_import(mock_func_factory, temp_d assert mock_run_legacy.last_args == ('update.php', '--recompute-importance') -@pytest.mark.parametrize("params,func", [ - (('--init', '--no-update-functions'), 'init_replication'), - (('--check-for-updates',), 'check_for_updates') - ]) -def test_replication_command(mock_func_factory, temp_db, params, func): - func_mock = mock_func_factory(nominatim.tools.replication, func) - - assert 0 == call_nominatim('replication', *params) - assert func_mock.called == 1 - - -def test_replication_update_bad_interval(monkeypatch, temp_db): - monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', 'xx') - - assert call_nominatim('replication') == 1 - - -def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db): - monkeypatch.setenv('NOMINATIM_REPLICATION_URL', - 'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates') - - assert call_nominatim('replication') == 1 - - -@pytest.mark.parametrize("state", [nominatim.tools.replication.UpdateState.UP_TO_DATE, - nominatim.tools.replication.UpdateState.NO_CHANGES]) -def test_replication_update_once_no_index(mock_func_factory, temp_db, temp_db_conn, - status_table, state): - status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1) - func_mock = mock_func_factory(nominatim.tools.replication, 'update') - - assert 0 == call_nominatim('replication', '--once', '--no-index') - - -def test_replication_update_continuous(monkeypatch, temp_db_conn, status_table): - status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1) - states = [nominatim.tools.replication.UpdateState.UP_TO_DATE, - nominatim.tools.replication.UpdateState.UP_TO_DATE] - monkeypatch.setattr(nominatim.tools.replication, 'update', - lambda *args, **kwargs: states.pop()) - - index_mock = MockParamCapture() - monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', index_mock) - monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', index_mock) - - with pytest.raises(IndexError): - call_nominatim('replication') - - assert index_mock.called == 4 - - -def test_replication_update_continuous_no_change(monkeypatch, temp_db_conn, status_table): - status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1) - states = [nominatim.tools.replication.UpdateState.NO_CHANGES, - nominatim.tools.replication.UpdateState.UP_TO_DATE] - monkeypatch.setattr(nominatim.tools.replication, 'update', - lambda *args, **kwargs: states.pop()) - - index_mock = MockParamCapture() - monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', index_mock) - monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', index_mock) - - sleep_mock = MockParamCapture() - monkeypatch.setattr(time, 'sleep', sleep_mock) - - with pytest.raises(IndexError): - call_nominatim('replication') - - assert index_mock.called == 2 - assert sleep_mock.called == 1 - assert sleep_mock.last_args[0] == 60 - - def test_serve_command(mock_func_factory): func = mock_func_factory(nominatim.cli, 'run_php_server') diff --git a/test/python/test_cli_replication.py b/test/python/test_cli_replication.py new file mode 100644 index 00000000..a62ad1a4 --- /dev/null +++ b/test/python/test_cli_replication.py @@ -0,0 +1,127 @@ +""" +Tests for replication command of command-line interface wrapper. +""" +import datetime as dt +import time +from pathlib import Path + +import pytest + +import nominatim.cli +import nominatim.indexer.indexer +import nominatim.tools.replication +from nominatim.db import status + +from mocks import MockParamCapture + +SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve() + +def call_nominatim(*args): + return nominatim.cli.nominatim(module_dir='build/module', + osm2pgsql_path='build/osm2pgsql/osm2pgsql', + phplib_dir=str(SRC_DIR / 'lib-php'), + data_dir=str(SRC_DIR / 'data'), + phpcgi_path='/usr/bin/php-cgi', + sqllib_dir=str(SRC_DIR / 'lib-sql'), + config_dir=str(SRC_DIR / 'settings'), + cli_args=['replication'] + list(args)) + +@pytest.fixture +def index_mock(monkeypatch): + mock = MockParamCapture() + monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', mock) + monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', mock) + + return mock + + +@pytest.fixture +def mock_func_factory(monkeypatch): + def get_mock(module, func): + mock = MockParamCapture() + monkeypatch.setattr(module, func, mock) + return mock + + return get_mock + + +@pytest.fixture +def init_status(temp_db_conn, status_table): + status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1) + return 1 + + +@pytest.fixture +def update_mock(mock_func_factory, init_status): + return mock_func_factory(nominatim.tools.replication, 'update') + +@pytest.mark.parametrize("params,func", [ + (('--init', '--no-update-functions'), 'init_replication'), + (('--check-for-updates',), 'check_for_updates') + ]) +def test_replication_command(mock_func_factory, temp_db, params, func): + func_mock = mock_func_factory(nominatim.tools.replication, func) + + assert 0 == call_nominatim(*params) + assert func_mock.called == 1 + + +def test_replication_update_bad_interval(monkeypatch, temp_db): + monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', 'xx') + + assert call_nominatim() == 1 + + +def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db): + monkeypatch.setenv('NOMINATIM_REPLICATION_URL', + 'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates') + + assert call_nominatim() == 1 + + +def test_replication_update_once_no_index(update_mock): + assert 0 == call_nominatim('--once', '--no-index') + + assert str(update_mock.last_args[1]['osm2pgsql']) == 'build/osm2pgsql/osm2pgsql' + + +def test_replication_update_custom_osm2pgsql(monkeypatch, update_mock): + monkeypatch.setenv('NOMINATIM_OSM2PGSQL_BINARY', '/secret/osm2pgsql') + assert 0 == call_nominatim('--once', '--no-index') + + assert str(update_mock.last_args[1]['osm2pgsql']) == '/secret/osm2pgsql' + + +def test_replication_update_custom_threads(update_mock): + assert 0 == call_nominatim('--once', '--no-index', '--threads', '4') + + assert update_mock.last_args[1]['threads'] == 4 + + +def test_replication_update_continuous(monkeypatch, init_status, index_mock): + states = [nominatim.tools.replication.UpdateState.UP_TO_DATE, + nominatim.tools.replication.UpdateState.UP_TO_DATE] + monkeypatch.setattr(nominatim.tools.replication, 'update', + lambda *args, **kwargs: states.pop()) + + with pytest.raises(IndexError): + call_nominatim() + + assert index_mock.called == 4 + + +def test_replication_update_continuous_no_change(monkeypatch, init_status, index_mock): + states = [nominatim.tools.replication.UpdateState.NO_CHANGES, + nominatim.tools.replication.UpdateState.UP_TO_DATE] + monkeypatch.setattr(nominatim.tools.replication, 'update', + lambda *args, **kwargs: states.pop()) + + sleep_mock = MockParamCapture() + monkeypatch.setattr(time, 'sleep', sleep_mock) + + with pytest.raises(IndexError): + call_nominatim() + + assert index_mock.called == 2 + assert sleep_mock.called == 1 + assert sleep_mock.last_args[0] == 60