simplify interface for adding tiger data

Also simplifies tests using existing fixtures.

diff --git a/nominatim/cli.py b/nominatim/cli.py
index 47b25c17862bc3a3fb2843ab79b04a07a3180b7e..55f51aac72dc7d6a2b2337b198f2966d991a30f6 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -148,11 +148,8 @@ class UpdateAddData:
     @staticmethod
     def run(args):
         if args.tiger_data:
     @staticmethod
     def run(args):
         if args.tiger_data:

-            return tiger_data.add_tiger_data(args.config.get_libpq_dsn(),
-                                             args.tiger_data,
-                                             args.threads or 1,
-                                             args.config,
-                                             args.sqllib_dir)
+            return tiger_data.add_tiger_data(args.tiger_data,
+                                             args.config, args.threads or 1)

 
         params = ['update.php']
         if args.file:
 
         params = ['update.php']
         if args.file:

diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py
index c1de3615f7f953a65984b2bf0d644552b4d30cf0..90789e79e6165cb5254ff7eb0bc2feb967053ab7 100644 (file)
--- a/nominatim/tools/tiger_data.py
+++ b/nominatim/tools/tiger_data.py
@@ -76,17 +76,17 @@ def handle_unregister_connection_pool(sel, place_threads):
             conn.close()
             place_threads -= 1
 
             conn.close()
             place_threads -= 1
 

-def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
-    """ Import tiger data from directory or tar file
+def add_tiger_data(data_dir, config, threads):
+    """ Import tiger data from directory or tar file `data dir`.

     """
     """

-
+    dsn = config.get_libpq_dsn()

     sql_files, tar = handle_tarfile_or_directory(data_dir)
 
     if not sql_files:
         return
 
     with connect(dsn) as conn:
     sql_files, tar = handle_tarfile_or_directory(data_dir)
 
     if not sql_files:
         return
 
     with connect(dsn) as conn:

-        sql = SQLPreprocessor(conn, config, sqllib_dir)
+        sql = SQLPreprocessor(conn, config, config.lib_dir.sql)

         sql.run_sql_file(conn, 'tiger_import_start.sql')
 
     # Reading sql_files and then for each file line handling
         sql.run_sql_file(conn, 'tiger_import_start.sql')
 
     # Reading sql_files and then for each file line handling


@@ -116,5 +116,5 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
     print('\n')
     LOG.warning("Creating indexes on Tiger data")
     with connect(dsn) as conn:
     print('\n')
     LOG.warning("Creating indexes on Tiger data")
     with connect(dsn) as conn:

-        sql = SQLPreprocessor(conn, config, sqllib_dir)
+        sql = SQLPreprocessor(conn, config, config.lib_dir.sql)

         sql.run_sql_file(conn, 'tiger_import_finish.sql')
         sql.run_sql_file(conn, 'tiger_import_finish.sql')

diff --git a/test/python/conftest.py b/test/python/conftest.py
index 0d1cd2f37168f8446182d030d2e26e4427922ead..2a2d212960f3b1ad0312c892638b5a5188abaa7c 100644 (file)
--- a/test/python/conftest.py
+++ b/test/python/conftest.py
@@ -126,7 +126,12 @@ def table_factory(temp_db_cursor):
 
 @pytest.fixture
 def def_config():
 
 @pytest.fixture
 def def_config():

-    return Configuration(None, SRC_DIR.resolve() / 'settings')
+    cfg = Configuration(None, SRC_DIR.resolve() / 'settings')
+    cfg.set_libdirs(module='.', osm2pgsql='.',
+                    php=SRC_DIR / 'lib-php',
+                    sql=SRC_DIR / 'lib-sql',
+                    data=SRC_DIR / 'data')
+    return cfg

 
 @pytest.fixture
 def src_dir():
 
 @pytest.fixture
 def src_dir():

diff --git a/test/python/test_tools_tiger_data.py b/test/python/test_tools_tiger_data.py
index 1366fe3ec06327997ef90598c86ef5ecda051450..f97a571330c62ddba87c0e2412dffdf202d7cd95 100644 (file)
--- a/test/python/test_tools_tiger_data.py
+++ b/test/python/test_tools_tiger_data.py
@@ -10,35 +10,31 @@ from nominatim.tools import tiger_data, database_import
 
 
 @pytest.mark.parametrize("threads", (1, 5))
 
 
 @pytest.mark.parametrize("threads", (1, 5))

-def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
-                        temp_db_cursor, threads, temp_db):
-    temp_db_cursor.execute('CREATE EXTENSION hstore')
-    temp_db_cursor.execute('CREATE EXTENSION postgis')
+def test_add_tiger_data(def_config, tmp_path, sql_preprocessor,
+                        temp_db_cursor, threads, temp_db_with_extensions):

     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1);
                           INSERT INTO non_existant_table values (1);""")
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1);
                           INSERT INTO non_existant_table values (1);""")

-    tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
+    tiger_data.add_tiger_data(str(tmp_path), def_config, threads)

 
     assert temp_db_cursor.table_rows('place') == 1
 
 
     assert temp_db_cursor.table_rows('place') == 1
 

+

 @pytest.mark.parametrize("threads", (1, 5))
 @pytest.mark.parametrize("threads", (1, 5))

-def test_add_tiger_data_bad_file(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
-                        temp_db_cursor, threads, temp_db):
-    temp_db_cursor.execute('CREATE EXTENSION hstore')
-    temp_db_cursor.execute('CREATE EXTENSION postgis')
+def test_add_tiger_data_bad_file(def_config, tmp_path, sql_preprocessor,
+                                 temp_db_cursor, threads, temp_db_with_extensions):

     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.txt'
     sqlfile.write_text("""Random text""")
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.txt'
     sqlfile.write_text("""Random text""")

-    tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
+    tiger_data.add_tiger_data(str(tmp_path), def_config, threads)

 
     assert temp_db_cursor.table_rows('place') == 0
 
 
     assert temp_db_cursor.table_rows('place') == 0
 

+

 @pytest.mark.parametrize("threads", (1, 5))
 @pytest.mark.parametrize("threads", (1, 5))

-def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
-                        temp_db_cursor, threads, temp_db, sql_preprocessor):
-    temp_db_cursor.execute('CREATE EXTENSION hstore')
-    temp_db_cursor.execute('CREATE EXTENSION postgis')
+def test_add_tiger_data_tarfile(def_config, tmp_path, temp_db_cursor,
+                                threads, temp_db_with_extensions, sql_preprocessor):

     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1);
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1);


@@ -46,21 +42,20 @@ def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
     tar = tarfile.open("sample.tar.gz", "w:gz")
     tar.add(sqlfile)
     tar.close()
     tar = tarfile.open("sample.tar.gz", "w:gz")
     tar.add(sqlfile)
     tar.close()

-    tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql')
-    
+    tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
+

     assert temp_db_cursor.table_rows('place') == 1
 
     assert temp_db_cursor.table_rows('place') == 1
 

+

 @pytest.mark.parametrize("threads", (1, 5))
 @pytest.mark.parametrize("threads", (1, 5))

-def test_add_tiger_data_bad_tarfile(dsn, src_dir, def_config, tmp_path,
-                        temp_db_cursor, threads, temp_db, sql_preprocessor):
-    temp_db_cursor.execute('CREATE EXTENSION hstore')
-    temp_db_cursor.execute('CREATE EXTENSION postgis')
+def test_add_tiger_data_bad_tarfile(def_config, tmp_path, temp_db_cursor, threads,
+                                    temp_db_with_extensions, sql_preprocessor):

     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.txt'
     sqlfile.write_text("""Random text""")
     tar = tarfile.open("sample.tar.gz", "w:gz")
     tar.add(sqlfile)
     tar.close()
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
     sqlfile = tmp_path / '1010.txt'
     sqlfile.write_text("""Random text""")
     tar = tarfile.open("sample.tar.gz", "w:gz")
     tar.add(sqlfile)
     tar.close()

-    tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql')
-    
-    assert temp_db_cursor.table_rows('place') == 0
\ No newline at end of file
+    tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
+
+    assert temp_db_cursor.table_rows('place') == 0