Merge pull request #3045 from biswajit-k/taginfo

[nominatim.git] / test / python / cli / test_cli.py

diff --git a/test/python/cli/test_cli.py b/test/python/cli/test_cli.py
index dbe17083470fc051a2f9eb350565ab82278d2780..d0e3307ebf93be5209e1102e2b7f3e6eac5e0315 100644 (file)
--- a/test/python/cli/test_cli.py
+++ b/test/python/cli/test_cli.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Tests for command line interface wrapper.
 
 """
 Tests for command line interface wrapper.
 


@@ -5,256 +11,140 @@ These tests just check that the various command line parameters route to the
 correct functionionality. They use a lot of monkeypatching to avoid executing
 the actual functions.
 """
 correct functionionality. They use a lot of monkeypatching to avoid executing
 the actual functions.
 """

+import importlib

 import pytest
 
 import pytest
 

-import nominatim.db.properties
-import nominatim.cli
-import nominatim.clicmd.api
-import nominatim.clicmd.refresh
-import nominatim.clicmd.admin
-import nominatim.clicmd.setup

 import nominatim.indexer.indexer
 import nominatim.indexer.indexer

-import nominatim.tools.admin

 import nominatim.tools.add_osm_data
 import nominatim.tools.add_osm_data

-import nominatim.tools.check_database
-import nominatim.tools.database_import
-import nominatim.tools.country_info

 import nominatim.tools.freeze
 import nominatim.tools.freeze

-import nominatim.tools.refresh
-import nominatim.tools.postcodes
-import nominatim.tokenizer.factory

 
 
 
 

-class TestCli:
+def test_cli_help(cli_call, capsys):
+    """ Running nominatim tool without arguments prints help.
+    """
+    assert cli_call() == 1

 
 

-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call):
-        self.call_nominatim = cli_call
-
-
-    def test_cli_help(self, capsys):
-        """ Running nominatim tool without arguments prints help.
-        """
-        assert self.call_nominatim() == 1
-
-        captured = capsys.readouterr()
-        assert captured.out.startswith('usage:')
-
-
-    @pytest.mark.parametrize("command,script", [
-                             (('export',), 'export')
-                             ])
-    def test_legacy_commands_simple(self, mock_run_legacy, command, script):
-        assert self.call_nominatim(*command) == 0
-
-        assert mock_run_legacy.called == 1
-        assert mock_run_legacy.last_args[0] == script + '.php'
-
-
-    @pytest.mark.parametrize("params", [('--warm', ),
-                                        ('--warm', '--reverse-only'),
-                                        ('--warm', '--search-only')])
-    def test_admin_command_legacy(self, mock_func_factory, params):
-        mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
-
-        assert self.call_nominatim('admin', *params) == 0
-
-        assert mock_run_legacy.called == 1
-
-
-    def test_admin_command_check_database(self, mock_func_factory):
-        mock = mock_func_factory(nominatim.tools.check_database, 'check_database')
+    captured = capsys.readouterr()
+    assert captured.out.startswith('usage:')

 
 

-        assert self.call_nominatim('admin', '--check-database') == 0
-        assert mock.called == 1
+def test_cli_version(cli_call, capsys):
+    """ Running nominatim tool --version prints a version string.
+    """
+    assert cli_call('--version') == 0

 
 

+    captured = capsys.readouterr()
+    assert captured.out.startswith('Nominatim version')

 
 

-    @pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc')])
-    def test_add_data_file_command(self, mock_func_factory, name, oid):
-        mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_data_from_file')
-        assert self.call_nominatim('add-data', '--' + name, str(oid)) == 0
+@pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc')])
+def test_cli_add_data_file_command(cli_call, mock_func_factory, name, oid):
+    mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_data_from_file')
+    assert cli_call('add-data', '--' + name, str(oid)) == 0

 
 

-        assert mock_run_legacy.called == 1
+    assert mock_run_legacy.called == 1

 
 
 
 

-    @pytest.mark.parametrize("name,oid", [('node', 12), ('way', 8), ('relation', 32)])
-    def test_add_data_object_command(self, mock_func_factory, name, oid):
-        mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_osm_object')
-        assert self.call_nominatim('add-data', '--' + name, str(oid)) == 0
+@pytest.mark.parametrize("name,oid", [('node', 12), ('way', 8), ('relation', 32)])
+def test_cli_add_data_object_command(cli_call, mock_func_factory, name, oid):
+    mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_osm_object')
+    assert cli_call('add-data', '--' + name, str(oid)) == 0

 
 

-        assert mock_run_legacy.called == 1
+    assert mock_run_legacy.called == 1

 
 
 
 

-    def test_serve_command(self, mock_func_factory):
-        func = mock_func_factory(nominatim.cli, 'run_php_server')

 
 

-        self.call_nominatim('serve')
+def test_cli_add_data_tiger_data(cli_call, cli_tokenizer_mock, mock_func_factory):
+    mock = mock_func_factory(nominatim.tools.tiger_data, 'add_tiger_data')

 
 

-        assert func.called == 1
-
-
-@pytest.mark.parametrize("params", [('search', '--query', 'new', '--polygon-output', 'svg', '--polygon-threshold', '0.1'),
-                                    ('search', '--city', 'Berlin', '--format', 'xml', '--lang', 'de'),
-                                    ('reverse', '--lat', '0', '--lon', '0',
-                                     '--polygon-output', 'svg', '--polygon-threshold', '0.1', '--format', 'json', '--lang', 'en'),
-                                    ('lookup', '--id', 'N1',
-                                     '--polygon-output', 'svg', '--polygon-threshold', '0.1', '--format', 'json', '--lang', 'en'),
-                                    ('details', '--node', '1'),
-                                    ('details', '--way', '1'),
-                                    ('details', '--relation', '1'),
-                                    ('details', '--place_id', '10001'),
-                                    ('status',)])
-class TestCliApiCall:
-
-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call):
-        self.call_nominatim = cli_call
-
-    def test_api_commands_simple(self, mock_func_factory, params, tmp_path):
-        (tmp_path / 'website').mkdir()
-        (tmp_path / 'website' / (params[0] + '.php')).write_text('')
-        mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
+    assert cli_call('add-data', '--tiger-data', 'somewhere') == 0

 
 

-        assert self.call_nominatim(*params, '--project-dir', str(tmp_path)) == 0
+    assert mock.called == 1

 
 

-        assert mock_run_api.called == 1
-        assert mock_run_api.last_args[0] == params[0]

 
 

+def test_cli_serve_php(cli_call, mock_func_factory):
+    func = mock_func_factory(nominatim.cli, 'run_php_server')

 
 

-    def test_bad_project_idr(self, mock_func_factory, params):
-        mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
+    cli_call('serve') == 0

 
 

-        assert self.call_nominatim(*params) == 1
+    assert func.called == 1

 
 
 
 

-class TestCliWithDb:
-
-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call, temp_db):
-        self.call_nominatim = cli_call
-
-
-    @pytest.fixture(autouse=True)
-    def setup_tokenizer_mock(self, monkeypatch):
-        class DummyTokenizer:
-            def __init__(self, *args, **kwargs):
-                self.update_sql_functions_called = False
-                self.finalize_import_called = False
-                self.update_statistics_called = False
+def test_cli_serve_sanic(cli_call, mock_func_factory):
+    mod = pytest.importorskip("sanic")
+    func = mock_func_factory(mod.Sanic, "run")

 
 

-            def update_sql_functions(self, *args):
-                self.update_sql_functions_called = True
+    cli_call('serve', '--engine', 'sanic') == 0

 
 

-            def finalize_import(self, *args):
-                self.finalize_import_called = True
+    assert func.called == 1

 
 

-            def update_statistics(self):
-                self.update_statistics_called = True

 
 

+def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory):
+    pytest.importorskip("starlette")
+    mod = pytest.importorskip("uvicorn")
+    func = mock_func_factory(mod, "run")

 
 

-        tok = DummyTokenizer()
-        monkeypatch.setattr(nominatim.tokenizer.factory, 'get_tokenizer_for_db',
-                            lambda *args: tok)
-        monkeypatch.setattr(nominatim.tokenizer.factory, 'create_tokenizer',
-                            lambda *args: tok)
+    cli_call('serve', '--engine', 'starlette', '--server', 'foobar:4545') == 0

 
 

-        self.tokenizer_mock = tok
+    assert func.called == 1
+    assert func.last_kwargs['host'] == 'foobar'
+    assert func.last_kwargs['port'] == 4545

 
 
 
 

-    def test_import_missing_file(self):
-        assert self.call_nominatim('import', '--osm-file', 'sfsafegwedgw.reh.erh') == 1
+def test_cli_serve_starlette_custom_server_bad_port(cli_call, mock_func_factory):
+    pytest.importorskip("starlette")
+    mod = pytest.importorskip("uvicorn")
+    func = mock_func_factory(mod, "run")

 
 

+    cli_call('serve', '--engine', 'starlette', '--server', 'foobar:45:45') == 1

 
 

-    def test_import_bad_file(self):
-        assert self.call_nominatim('import', '--osm-file', '.') == 1

 
 

+@pytest.mark.parametrize("engine", ['falcon', 'starlette'])
+def test_cli_serve_uvicorn_based(cli_call, engine, mock_func_factory):
+    pytest.importorskip(engine)
+    mod = pytest.importorskip("uvicorn")
+    func = mock_func_factory(mod, "run")

 
 

-    def test_import_full(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'),
-            mock_func_factory(nominatim.tools.country_info, 'setup_country_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
-            mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
-            mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'load_data'),
-            mock_func_factory(nominatim.tools.database_import, 'create_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
-            mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_config'),
-            mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
+    cli_call('serve', '--engine', engine) == 0

 
 

-        cf_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
+    assert func.called == 1
+    assert func.last_kwargs['host'] == '127.0.0.1'
+    assert func.last_kwargs['port'] == 8088

 
 

-        assert self.call_nominatim('import', '--osm-file', __file__) == 0
-        assert self.tokenizer_mock.finalize_import_called
+def test_cli_export_command(cli_call, mock_run_legacy):
+    assert cli_call('export', '--output-all-postcodes') == 0

 
 

-        assert cf_mock.called > 1
+    assert mock_run_legacy.called == 1
+    assert mock_run_legacy.last_args[0] == 'export.php'

 
 

-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)

 
 

+@pytest.mark.parametrize("param,value", [('output-type', 'country'),
+                                         ('output-format', 'street;city'),
+                                         ('language', 'xf'),
+                                         ('restrict-to-country', 'us'),
+                                         ('restrict-to-osm-node', '536'),
+                                         ('restrict-to-osm-way', '727'),
+                                         ('restrict-to-osm-relation', '197532')
+                                        ])
+def test_export_parameters(src_dir, tmp_path, param, value, monkeypatch):
+    (tmp_path / 'admin').mkdir()
+    (tmp_path / 'admin' / 'export.php').write_text(f"""<?php
+        exit(strpos(implode(' ', $_SERVER['argv']), '--{param} {value}') >= 0 ? 0 : 10);
+        """)

 
 

-    def test_import_continue_load_data(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'load_data'),
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
+    monkeypatch.setattr(nominatim.paths, 'PHPLIB_DIR', tmp_path)

 
 

-        assert self.call_nominatim('import', '--continue', 'load-data') == 0
-        assert self.tokenizer_mock.finalize_import_called
+    assert nominatim.cli.nominatim(module_dir='MODULE NOT AVAILABLE',
+                                   osm2pgsql_path='OSM2PGSQL NOT AVAILABLE',
+                                   phpcgi_path='/usr/bin/php-cgi',
+                                   cli_args=['export', '--' + param, value]) == 0

 
 

-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)

 
 
 
 

-    def test_import_continue_indexing(self, mock_func_factory, placex_table,
-                                      temp_db_conn):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'indexing') == 0
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
-
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
-
-        # Calling it again still works for the index
-        assert self.call_nominatim('import', '--continue', 'indexing') == 0
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
-
-
-    def test_import_continue_postprocess(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'db-postprocess') == 0
-
-        assert self.tokenizer_mock.finalize_import_called
+class TestCliWithDb:

 
 

-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
+    @pytest.fixture(autouse=True)
+    def setup_cli_call(self, cli_call, temp_db, cli_tokenizer_mock):
+        self.call_nominatim = cli_call
+        self.tokenizer_mock = cli_tokenizer_mock

 
 
     def test_freeze_command(self, mock_func_factory):
 
 
     def test_freeze_command(self, mock_func_factory):


@@ -267,15 +157,6 @@ class TestCliWithDb:
         assert mock_flatnode.called == 1
 
 
         assert mock_flatnode.called == 1
 
 

-
-    @pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))])
-    def test_admin_command_tool(self, mock_func_factory, func, params):
-        mock = mock_func_factory(nominatim.tools.admin, func)
-
-        assert self.call_nominatim('admin', *params) == 0
-        assert mock.called == 1
-
-

     @pytest.mark.parametrize("params,do_bnds,do_ranks", [
                               ([], 1, 1),
                               (['--boundaries-only'], 1, 0),
     @pytest.mark.parametrize("params,do_bnds,do_ranks", [
                               ([], 1, 1),
                               (['--boundaries-only'], 1, 0),


@@ -292,71 +173,27 @@ class TestCliWithDb:
         assert bnd_mock.called == do_bnds
         assert rank_mock.called == do_ranks
 
         assert bnd_mock.called == do_bnds
         assert rank_mock.called == do_ranks
 

-    @pytest.mark.parametrize("no_replace", [(True), (False)])
-    def test_special_phrases_wiki_command(self, mock_func_factory, no_replace):
+
+    def test_special_phrases_wiki_command(self, mock_func_factory):

         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
 
         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
 

-        if no_replace:
-            self.call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')
-        else:
-            self.call_nominatim('special-phrases', '--import-from-wiki')
+        self.call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')

 
         assert func.called == 1
 
 
         assert func.called == 1
 

-    @pytest.mark.parametrize("no_replace", [(True), (False)])
-    def test_special_phrases_csv_command(self, src_dir, mock_func_factory, no_replace):
+
+    def test_special_phrases_csv_command(self, src_dir, mock_func_factory):

         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
         testdata = src_dir / 'test' / 'testdb'
         csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
 
         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
         testdata = src_dir / 'test' / 'testdb'
         csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
 

-        if no_replace:
-            self.call_nominatim('special-phrases', '--import-from-csv', csv_path, '--no-replace')
-        else:
-            self.call_nominatim('special-phrases', '--import-from-csv', csv_path)
+        self.call_nominatim('special-phrases', '--import-from-csv', csv_path)

 
         assert func.called == 1
 
 
         assert func.called == 1
 

-    @pytest.mark.parametrize("command,func", [
-                             ('address-levels', 'load_address_levels_from_config'),
-                             ('wiki-data', 'import_wikipedia_articles'),
-                             ('importance', 'recompute_importance'),
-                             ('website', 'setup_website'),
-                             ])
-    def test_refresh_command(self, mock_func_factory, command, func):
-        func_mock = mock_func_factory(nominatim.tools.refresh, func)
-
-        assert self.call_nominatim('refresh', '--' + command) == 0
-        assert func_mock.called == 1
-
-
-    def test_refresh_word_count(self):
-        assert self.call_nominatim('refresh', '--word-count') == 0
-        assert self.tokenizer_mock.update_statistics_called
-
-
-    def test_refresh_postcodes(self, mock_func_factory, place_table):
-        func_mock = mock_func_factory(nominatim.tools.postcodes, 'update_postcodes')
-        idx_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_postcodes')
-
-        assert self.call_nominatim('refresh', '--postcodes') == 0
-        assert func_mock.called == 1
-        assert idx_mock.called == 1
-
-    def test_refresh_create_functions(self, mock_func_factory):
-        func_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
-
-        assert self.call_nominatim('refresh', '--functions') == 0
-        assert func_mock.called == 1
-        assert self.tokenizer_mock.update_sql_functions_called
-
-
-    def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
-        calls = []
-        monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
-                            lambda *args, **kwargs: calls.append('import') or 0)
-        monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance',
-                            lambda *args, **kwargs: calls.append('update'))

 
 

-        assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0
+    def test_special_phrases_csv_bad_file(self, src_dir):
+        testdata = src_dir / 'something349053905.csv'

 
 

-        assert calls == ['import', 'update']
+        self.call_nominatim('special-phrases', '--import-from-csv',
+                            str(testdata.resolve())) == 1