Added fixture for sql_preprocessor and fixed some issues

diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py
index e3adfd33a760c995ca25c53c057e5b5a00c8ee7d..9b960e2d54ee82df6f05c921ccb44062f33d9ec6 100644 (file)
--- a/nominatim/tools/tiger_data.py
+++ b/nominatim/tools/tiger_data.py
@@ -44,9 +44,7 @@ def handle_threaded_sql_statements(sel, file):
     lines = 0
     end_of_file = False
     # Using pool of database connections to execute sql statements
     lines = 0
     end_of_file = False
     # Using pool of database connections to execute sql statements

-    while True:
-        if end_of_file:
-            break
+    while not end_of_file:

         for key, _ in sel.select(1):
             conn = key.data
             try:
         for key, _ in sel.select(1):
             conn = key.data
             try:


@@ -61,7 +59,7 @@ def handle_threaded_sql_statements(sel, file):
                         print('. ', end='', flush=True)
                         lines = 0
             except Exception as exc: # pylint: disable=broad-except
                         print('. ', end='', flush=True)
                         lines = 0
             except Exception as exc: # pylint: disable=broad-except

-                LOG.error('Wrong SQL statement: %s', exc)
+                LOG.info('Wrong SQL statement: %s', exc)

 
 
 def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
 
 
 def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):

diff --git a/test/python/conftest.py b/test/python/conftest.py
index d16dceffccb1008a1fe6082ab0a5e9d04cd2a439..4b7cccc39f09992b73c48d348ebd150a4942d8b9 100644 (file)
--- a/test/python/conftest.py
+++ b/test/python/conftest.py
@@ -13,6 +13,7 @@ sys.path.insert(0, str(SRC_DIR.resolve()))
 
 from nominatim.config import Configuration
 from nominatim.db import connection
 
 from nominatim.config import Configuration
 from nominatim.db import connection

+from nominatim.db.sql_preprocessor import SQLPreprocessor

 
 class _TestingCursor(psycopg2.extras.DictCursor):
     """ Extension to the DictCursor class that provides execution
 
 class _TestingCursor(psycopg2.extras.DictCursor):
     """ Extension to the DictCursor class that provides execution


@@ -266,3 +267,9 @@ def osm2pgsql_options(temp_db):
                 flatnode_file='',
                 tablespaces=dict(slim_data='', slim_index='',
                                  main_data='', main_index=''))
                 flatnode_file='',
                 tablespaces=dict(slim_data='', slim_index='',
                                  main_data='', main_index=''))

+
+@pytest.fixture
+def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
+    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
+    table_factory('country_name', 'partition INT', (0, 1, 2))
+    return SQLPreprocessor(temp_db_conn, def_config, tmp_path)

diff --git a/test/python/test_db_sql_preprocessor.py b/test/python/test_db_sql_preprocessor.py
index 3c10000f9169c883f56547665fef1e171f27ba90..08a195bd2d794d9c6f9f01cd18116df64d466b42 100644 (file)
--- a/test/python/test_db_sql_preprocessor.py
+++ b/test/python/test_db_sql_preprocessor.py
@@ -5,8 +5,6 @@ from pathlib import Path
 
 import pytest
 
 
 import pytest
 

-from nominatim.db.sql_preprocessor import SQLPreprocessor
-

 @pytest.fixture
 def sql_factory(tmp_path):
     def _mk_sql(sql_body):
 @pytest.fixture
 def sql_factory(tmp_path):
     def _mk_sql(sql_body):


@@ -21,13 +19,6 @@ def sql_factory(tmp_path):
 
     return _mk_sql
 
 
     return _mk_sql
 

-
-@pytest.fixture
-def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
-    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
-    table_factory('country_name', 'partition INT', (0, 1, 2))
-    return SQLPreprocessor(temp_db_conn, def_config, tmp_path)
-

 @pytest.mark.parametrize("expr,ret", [
     ("'a'", 'a'),
     ("'{{db.partitions|join}}'", '012'),
 @pytest.mark.parametrize("expr,ret", [
     ("'a'", 'a'),
     ("'{{db.partitions|join}}'", '012'),

diff --git a/test/python/test_tools_tiger_data.py b/test/python/test_tools_tiger_data.py
index 6290b9944d071267e06a6228993c63f22d1b034d..5029a1320c9d36c5df426660a2e84e143c9d0024 100644 (file)
--- a/test/python/test_tools_tiger_data.py
+++ b/test/python/test_tools_tiger_data.py
@@ -10,15 +10,11 @@ from nominatim.tools import tiger_data, database_import
 
 
 @pytest.mark.parametrize("threads", (1, 5))
 
 
 @pytest.mark.parametrize("threads", (1, 5))

-def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path,
+def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor,

                         temp_db_cursor, threads, temp_db):
                         temp_db_cursor, threads, temp_db):

-    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')

     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')
     temp_db_cursor.execute('CREATE TABLE place (id INT)')

-
-    database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
-                                     ignore_partitions=False)

     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1)""")
     tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1)""")
     tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')


@@ -26,15 +22,11 @@ def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path,
     assert temp_db_cursor.table_rows('place') == 1
 
 @pytest.mark.parametrize("threads", (1, 5))
     assert temp_db_cursor.table_rows('place') == 1
 
 @pytest.mark.parametrize("threads", (1, 5))

-def test_add_tiger_data_tarfile(dsn, src_dir, def_config, monkeypatch,tmp_path,
-                        temp_db_cursor, threads, temp_db):
-    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
+def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
+                        temp_db_cursor, threads, temp_db, sql_preprocessor):

     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')
     temp_db_cursor.execute('CREATE TABLE place (id INT)')

-
-    database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
-                                     ignore_partitions=False)

     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1)""")
     tar = tarfile.open("sample.tar.gz", "w:gz")
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1)""")
     tar = tarfile.open("sample.tar.gz", "w:gz")