Merge remote-tracking branch 'upstream/master'

[nominatim.git] / test / python / cli / test_cli.py

diff --git a/test/python/cli/test_cli.py b/test/python/cli/test_cli.py
index f4878b6186feaab16237a5169a528da074f5fb10..2831f84f1bc801d690966827bc2f741f4a440471 100644 (file)
--- a/test/python/cli/test_cli.py
+++ b/test/python/cli/test_cli.py
@@ -1,262 +1,88 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2024 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Tests for command line interface wrapper.
 
 These tests just check that the various command line parameters route to the
 """
 Tests for command line interface wrapper.
 
 These tests just check that the various command line parameters route to the

-correct functionionality. They use a lot of monkeypatching to avoid executing
+correct functionality. They use a lot of monkeypatching to avoid executing

 the actual functions.
 """
 the actual functions.
 """

+import importlib

 import pytest
 
 import pytest
 

-import nominatim.db.properties
-import nominatim.cli
-import nominatim.clicmd.api
-import nominatim.clicmd.refresh
-import nominatim.clicmd.admin
-import nominatim.clicmd.setup
-import nominatim.indexer.indexer
-import nominatim.tools.admin
-import nominatim.tools.add_osm_data
-import nominatim.tools.check_database
-import nominatim.tools.database_import
-import nominatim.tools.country_info
-import nominatim.tools.freeze
-import nominatim.tools.refresh
-import nominatim.tools.postcodes
-import nominatim.tokenizer.factory
-
-
-class TestCli:
+import nominatim_db.indexer.indexer
+import nominatim_db.tools.add_osm_data
+import nominatim_db.tools.freeze
+import nominatim_db.tools.tiger_data

 
 

-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call):
-        self.call_nominatim = cli_call

 
 

+def test_cli_help(cli_call, capsys):
+    """ Running nominatim tool without arguments prints help.
+    """
+    assert cli_call() == 1

 
 

-    def test_cli_help(self, capsys):
-        """ Running nominatim tool without arguments prints help.
-        """
-        assert self.call_nominatim() == 1
+    captured = capsys.readouterr()
+    assert captured.out.startswith('usage:')

 
 

-        captured = capsys.readouterr()
-        assert captured.out.startswith('usage:')
+def test_cli_version(cli_call, capsys):
+    """ Running nominatim tool --version prints a version string.
+    """
+    assert cli_call('--version') == 0

 
 

+    captured = capsys.readouterr()
+    assert captured.out.startswith('Nominatim version')

 
 

-    @pytest.mark.parametrize("command,script", [
-                             (('export',), 'export')
-                             ])
-    def test_legacy_commands_simple(self, mock_run_legacy, command, script):
-        assert self.call_nominatim(*command) == 0

 
 

-        assert mock_run_legacy.called == 1
-        assert mock_run_legacy.last_args[0] == script + '.php'
+def test_cli_serve_php(cli_call, mock_func_factory):
+    func = mock_func_factory(nominatim_db.cli, 'run_php_server')

 
 

+    cli_call('serve', '--engine', 'php') == 0

 
 

-    @pytest.mark.parametrize("params", [('--warm', ),
-                                        ('--warm', '--reverse-only'),
-                                        ('--warm', '--search-only')])
-    def test_admin_command_legacy(self, mock_func_factory, params):
-        mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
+    assert func.called == 1

 
 

-        assert self.call_nominatim('admin', *params) == 0
-
-        assert mock_run_legacy.called == 1

 
 
 
 

-    def test_admin_command_check_database(self, mock_func_factory):
-        mock = mock_func_factory(nominatim.tools.check_database, 'check_database')
+class TestCliWithDb:

 
 

-        assert self.call_nominatim('admin', '--check-database') == 0
-        assert mock.called == 1
+    @pytest.fixture(autouse=True)
+    def setup_cli_call(self, cli_call, temp_db, cli_tokenizer_mock, table_factory):
+        self.call_nominatim = cli_call
+        self.tokenizer_mock = cli_tokenizer_mock
+        # Make sure tools.freeze.is_frozen doesn't report database as frozen. Monkeypatching failed
+        table_factory('place')

 
 
     @pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc')])
 
 
     @pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc')])

-    def test_add_data_file_command(self, mock_func_factory, name, oid):
-        mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_data_from_file')
-        assert self.call_nominatim('add-data', '--' + name, str(oid)) == 0
+    def test_cli_add_data_file_command(self, cli_call, mock_func_factory, name, oid):
+        mock_run_legacy = mock_func_factory(nominatim_db.tools.add_osm_data, 'add_data_from_file')
+        assert cli_call('add-data', '--' + name, str(oid)) == 0

 
         assert mock_run_legacy.called == 1
 
 
     @pytest.mark.parametrize("name,oid", [('node', 12), ('way', 8), ('relation', 32)])
 
         assert mock_run_legacy.called == 1
 
 
     @pytest.mark.parametrize("name,oid", [('node', 12), ('way', 8), ('relation', 32)])

-    def test_add_data_object_command(self, mock_func_factory, name, oid):
-        mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_osm_object')
-        assert self.call_nominatim('add-data', '--' + name, str(oid)) == 0
+    def test_cli_add_data_object_command(self, cli_call, mock_func_factory, name, oid):
+        mock_run_legacy = mock_func_factory(nominatim_db.tools.add_osm_data, 'add_osm_object')
+        assert cli_call('add-data', '--' + name, str(oid)) == 0

 
         assert mock_run_legacy.called == 1
 
 
 
         assert mock_run_legacy.called == 1
 
 

-    def test_serve_command(self, mock_func_factory):
-        func = mock_func_factory(nominatim.cli, 'run_php_server')
-
-        self.call_nominatim('serve')
-
-        assert func.called == 1
-
-
-@pytest.mark.parametrize("params", [('search', '--query', 'new'),
-                                    ('reverse', '--lat', '0', '--lon', '0'),
-                                    ('lookup', '--id', 'N1'),
-                                    ('details', '--node', '1'),
-                                    ('details', '--way', '1'),
-                                    ('details', '--relation', '1'),
-                                    ('details', '--place_id', '10001'),
-                                    ('status',)])
-class TestCliApiCall:
-
-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call):
-        self.call_nominatim = cli_call
-
-    def test_api_commands_simple(self, mock_func_factory, params, tmp_path):
-        (tmp_path / 'website').mkdir()
-        (tmp_path / 'website' / (params[0] + '.php')).write_text('')
-        mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
-
-        assert self.call_nominatim(*params, '--project-dir', str(tmp_path)) == 0
-
-        assert mock_run_api.called == 1
-        assert mock_run_api.last_args[0] == params[0]
-
-
-    def test_bad_project_idr(self, mock_func_factory, params):
-        mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
-
-        assert self.call_nominatim(*params) == 1
-
-
-class TestCliWithDb:
-
-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call, temp_db):
-        self.call_nominatim = cli_call
-
-
-    @pytest.fixture(autouse=True)
-    def setup_tokenizer_mock(self, monkeypatch):
-        class DummyTokenizer:
-            def __init__(self, *args, **kwargs):
-                self.update_sql_functions_called = False
-                self.finalize_import_called = False
-                self.update_statistics_called = False
-
-            def update_sql_functions(self, *args):
-                self.update_sql_functions_called = True
-
-            def finalize_import(self, *args):
-                self.finalize_import_called = True
-
-            def update_statistics(self):
-                self.update_statistics_called = True
-
-
-        tok = DummyTokenizer()
-        monkeypatch.setattr(nominatim.tokenizer.factory, 'get_tokenizer_for_db',
-                            lambda *args: tok)
-        monkeypatch.setattr(nominatim.tokenizer.factory, 'create_tokenizer',
-                            lambda *args: tok)
-
-        self.tokenizer_mock = tok
-
-
-    def test_import_missing_file(self):
-        assert self.call_nominatim('import', '--osm-file', 'sfsafegwedgw.reh.erh') == 1

 
 

+    def test_cli_add_data_tiger_data(self, cli_call, cli_tokenizer_mock, async_mock_func_factory):
+        mock = async_mock_func_factory(nominatim_db.tools.tiger_data, 'add_tiger_data')

 
 

-    def test_import_bad_file(self):
-        assert self.call_nominatim('import', '--osm-file', '.') == 1
-
-
-    def test_import_full(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'),
-            mock_func_factory(nominatim.tools.country_info, 'setup_country_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
-            mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
-            mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'load_data'),
-            mock_func_factory(nominatim.tools.database_import, 'create_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
-            mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_config'),
-            mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        cf_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
-
-        assert self.call_nominatim('import', '--osm-file', __file__) == 0
-        assert self.tokenizer_mock.finalize_import_called
-
-        assert cf_mock.called > 1
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
-
-
-    def test_import_continue_load_data(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'load_data'),
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'load-data') == 0
-        assert self.tokenizer_mock.finalize_import_called
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
-
-
-    def test_import_continue_indexing(self, mock_func_factory, placex_table,
-                                      temp_db_conn):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'indexing') == 0
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
-
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
-
-        # Calling it again still works for the index
-        assert self.call_nominatim('import', '--continue', 'indexing') == 0
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
-
-
-    def test_import_continue_postprocess(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'db-postprocess') == 0
-
-        assert self.tokenizer_mock.finalize_import_called
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
+        assert cli_call('add-data', '--tiger-data', 'somewhere') == 0

 
 

+        assert mock.called == 1

 
     def test_freeze_command(self, mock_func_factory):
 
     def test_freeze_command(self, mock_func_factory):

-        mock_drop = mock_func_factory(nominatim.tools.freeze, 'drop_update_tables')
-        mock_flatnode = mock_func_factory(nominatim.tools.freeze, 'drop_flatnode_file')
+        mock_drop = mock_func_factory(nominatim_db.tools.freeze, 'drop_update_tables')
+        mock_flatnode = mock_func_factory(nominatim_db.tools.freeze, 'drop_flatnode_file')

 
         assert self.call_nominatim('freeze') == 0
 
 
         assert self.call_nominatim('freeze') == 0
 


@@ -264,96 +90,48 @@ class TestCliWithDb:
         assert mock_flatnode.called == 1
 
 
         assert mock_flatnode.called == 1
 
 

-
-    @pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))])
-    def test_admin_command_tool(self, mock_func_factory, func, params):
-        mock = mock_func_factory(nominatim.tools.admin, func)
-
-        assert self.call_nominatim('admin', *params) == 0
-        assert mock.called == 1
-
-

     @pytest.mark.parametrize("params,do_bnds,do_ranks", [
     @pytest.mark.parametrize("params,do_bnds,do_ranks", [

-                              ([], 1, 1),
-                              (['--boundaries-only'], 1, 0),
-                              (['--no-boundaries'], 0, 1),
+                              ([], 2, 2),
+                              (['--boundaries-only'], 2, 0),
+                              (['--no-boundaries'], 0, 2),

                               (['--boundaries-only', '--no-boundaries'], 0, 0)])
                               (['--boundaries-only', '--no-boundaries'], 0, 0)])

-    def test_index_command(self, mock_func_factory, table_factory,
+    def test_index_command(self, monkeypatch, async_mock_func_factory, table_factory,

                            params, do_bnds, do_ranks):
         table_factory('import_status', 'indexed bool')
                            params, do_bnds, do_ranks):
         table_factory('import_status', 'indexed bool')

-        bnd_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_boundaries')
-        rank_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_by_rank')
+        bnd_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_boundaries')
+        rank_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_by_rank')
+        postcode_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_postcodes')
+
+        monkeypatch.setattr(nominatim_db.indexer.indexer.Indexer, 'has_pending', 
+                            [False, True].pop)

 
         assert self.call_nominatim('index', *params) == 0
 
         assert bnd_mock.called == do_bnds
         assert rank_mock.called == do_ranks
 
         assert self.call_nominatim('index', *params) == 0
 
         assert bnd_mock.called == do_bnds
         assert rank_mock.called == do_ranks

+        assert postcode_mock.called == do_ranks

 
 

-    @pytest.mark.parametrize("no_replace", [(True), (False)])
-    def test_special_phrases_wiki_command(self, mock_func_factory, no_replace):
-        func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')

 
 

-        if no_replace:
-            self.call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')
-        else:
-            self.call_nominatim('special-phrases', '--import-from-wiki')
+    def test_special_phrases_wiki_command(self, mock_func_factory):
+        func = mock_func_factory(nominatim_db.clicmd.special_phrases.SPImporter, 'import_phrases')
+
+        self.call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')

 
         assert func.called == 1
 
 
         assert func.called == 1
 

-    @pytest.mark.parametrize("no_replace", [(True), (False)])
-    def test_special_phrases_csv_command(self, src_dir, mock_func_factory, no_replace):
-        func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
+
+    def test_special_phrases_csv_command(self, src_dir, mock_func_factory):
+        func = mock_func_factory(nominatim_db.clicmd.special_phrases.SPImporter, 'import_phrases')

         testdata = src_dir / 'test' / 'testdb'
         csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
 
         testdata = src_dir / 'test' / 'testdb'
         csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
 

-        if no_replace:
-            self.call_nominatim('special-phrases', '--import-from-csv', csv_path, '--no-replace')
-        else:
-            self.call_nominatim('special-phrases', '--import-from-csv', csv_path)
+        self.call_nominatim('special-phrases', '--import-from-csv', csv_path)

 
         assert func.called == 1
 
 
         assert func.called == 1
 

-    @pytest.mark.parametrize("command,func", [
-                             ('address-levels', 'load_address_levels_from_config'),
-                             ('wiki-data', 'import_wikipedia_articles'),
-                             ('importance', 'recompute_importance'),
-                             ('website', 'setup_website'),
-                             ])
-    def test_refresh_command(self, mock_func_factory, command, func):
-        func_mock = mock_func_factory(nominatim.tools.refresh, func)
-
-        assert self.call_nominatim('refresh', '--' + command) == 0
-        assert func_mock.called == 1
-
-
-    def test_refresh_word_count(self):
-        assert self.call_nominatim('refresh', '--word-count') == 0
-        assert self.tokenizer_mock.update_statistics_called
-
-
-    def test_refresh_postcodes(self, mock_func_factory, place_table):
-        func_mock = mock_func_factory(nominatim.tools.postcodes, 'update_postcodes')
-        idx_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_postcodes')
-
-        assert self.call_nominatim('refresh', '--postcodes') == 0
-        assert func_mock.called == 1
-        assert idx_mock.called == 1
-
-    def test_refresh_create_functions(self, mock_func_factory):
-        func_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
-
-        assert self.call_nominatim('refresh', '--functions') == 0
-        assert func_mock.called == 1
-        assert self.tokenizer_mock.update_sql_functions_called
-
-
-    def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
-        calls = []
-        monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
-                            lambda *args, **kwargs: calls.append('import') or 0)
-        monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance',
-                            lambda *args, **kwargs: calls.append('update'))

 
 

-        assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0
+    def test_special_phrases_csv_bad_file(self, src_dir):
+        testdata = src_dir / 'something349053905.csv'

 
 

-        assert calls == ['import', 'update']
+        self.call_nominatim('special-phrases', '--import-from-csv',
+                            str(testdata.resolve())) == 1