From 4fa6c0ad5333e979931d3276a5a9cdcbfd514430 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 19 Apr 2021 09:38:17 +0200 Subject: [PATCH] simplify constructor for SQL preprocessor Use sql path from config. --- nominatim/db/sql_preprocessor.py | 4 ++-- nominatim/tools/database_import.py | 8 ++++---- nominatim/tools/refresh.py | 2 +- nominatim/tools/tiger_data.py | 4 ++-- test/python/conftest.py | 8 ++++++-- .../test_tools_refresh_create_functions.py | 17 +++++++++++------ 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/nominatim/db/sql_preprocessor.py b/nominatim/db/sql_preprocessor.py index 7ffe8881..c7009b34 100644 --- a/nominatim/db/sql_preprocessor.py +++ b/nominatim/db/sql_preprocessor.py @@ -75,9 +75,9 @@ class SQLPreprocessor: # pylint: disable=too-few-public-methods and follows its syntax. """ - def __init__(self, conn, config, sqllib_dir): + def __init__(self, conn, config): self.env = jinja2.Environment(autoescape=False, - loader=jinja2.FileSystemLoader(str(sqllib_dir))) + loader=jinja2.FileSystemLoader(str(config.lib_dir.sql))) db_info = {} db_info['partitions'] = _get_partitions(conn) diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index 964bc702..c8a4e51e 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -184,7 +184,7 @@ def create_tables(conn, config, sqllib_dir, reverse_only=False): When `reverse_only` is True, then the main table for searching will be skipped and only reverse search is possible. """ - sql = SQLPreprocessor(conn, config, sqllib_dir) + sql = SQLPreprocessor(conn, config) sql.env.globals['db']['reverse_only'] = reverse_only sql.run_sql_file(conn, 'tables.sql') @@ -194,14 +194,14 @@ def create_table_triggers(conn, config, sqllib_dir): """ Create the triggers for the tables. The trigger functions must already have been imported with refresh.create_functions(). """ - sql = SQLPreprocessor(conn, config, sqllib_dir) + sql = SQLPreprocessor(conn, config) sql.run_sql_file(conn, 'table-triggers.sql') def create_partition_tables(conn, config, sqllib_dir): """ Create tables that have explicit partitioning. """ - sql = SQLPreprocessor(conn, config, sqllib_dir) + sql = SQLPreprocessor(conn, config) sql.run_sql_file(conn, 'partition-tables.src.sql') @@ -303,7 +303,7 @@ def create_search_indices(conn, config, sqllib_dir, drop=False): cur.execute('DROP INDEX "{}"'.format(idx)) conn.commit() - sql = SQLPreprocessor(conn, config, sqllib_dir) + sql = SQLPreprocessor(conn, config) sql.run_sql_file(conn, 'indices.sql', drop=drop) diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index 77eecf04..45aa1504 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -81,7 +81,7 @@ def create_functions(conn, config, sqllib_dir, enable_diff_updates=True, enable_debug=False): """ (Re)create the PL/pgSQL functions. """ - sql = SQLPreprocessor(conn, config, sqllib_dir) + sql = SQLPreprocessor(conn, config) sql.run_sql_file(conn, 'functions.sql', disable_diff_updates=not enable_diff_updates, diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py index 90789e79..07772c70 100644 --- a/nominatim/tools/tiger_data.py +++ b/nominatim/tools/tiger_data.py @@ -86,7 +86,7 @@ def add_tiger_data(data_dir, config, threads): return with connect(dsn) as conn: - sql = SQLPreprocessor(conn, config, config.lib_dir.sql) + sql = SQLPreprocessor(conn, config) sql.run_sql_file(conn, 'tiger_import_start.sql') # Reading sql_files and then for each file line handling @@ -116,5 +116,5 @@ def add_tiger_data(data_dir, config, threads): print('\n') LOG.warning("Creating indexes on Tiger data") with connect(dsn) as conn: - sql = SQLPreprocessor(conn, config, config.lib_dir.sql) + sql = SQLPreprocessor(conn, config) sql.run_sql_file(conn, 'tiger_import_finish.sql') diff --git a/test/python/conftest.py b/test/python/conftest.py index 2a2d2129..4b9749c0 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -280,7 +280,11 @@ def osm2pgsql_options(temp_db): main_data='', main_index='')) @pytest.fixture -def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory): +def sql_preprocessor(temp_db_conn, tmp_path, monkeypatch, table_factory): monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') table_factory('country_name', 'partition INT', (0, 1, 2)) - return SQLPreprocessor(temp_db_conn, def_config, tmp_path) + cfg = Configuration(None, SRC_DIR.resolve() / 'settings') + cfg.set_libdirs(module='.', osm2pgsql='.', php=SRC_DIR / 'lib-php', + sql=tmp_path, data=SRC_DIR / 'data') + + return SQLPreprocessor(temp_db_conn, cfg) diff --git a/test/python/test_tools_refresh_create_functions.py b/test/python/test_tools_refresh_create_functions.py index 40d4c81a..53641cf9 100644 --- a/test/python/test_tools_refresh_create_functions.py +++ b/test/python/test_tools_refresh_create_functions.py @@ -5,6 +5,11 @@ import pytest from nominatim.tools.refresh import create_functions +@pytest.fixture +def sql_tmp_path(tmp_path, def_config): + def_config.lib_dir.sql = tmp_path + return tmp_path + @pytest.fixture def conn(temp_db_conn, table_factory, monkeypatch): monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') @@ -12,8 +17,8 @@ def conn(temp_db_conn, table_factory, monkeypatch): return temp_db_conn -def test_create_functions(temp_db_cursor, conn, def_config, tmp_path): - sqlfile = tmp_path / 'functions.sql' +def test_create_functions(temp_db_cursor, conn, def_config, sql_tmp_path): + sqlfile = sql_tmp_path / 'functions.sql' sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER AS $$ BEGIN @@ -22,14 +27,14 @@ def test_create_functions(temp_db_cursor, conn, def_config, tmp_path): $$ LANGUAGE plpgsql IMMUTABLE; """) - create_functions(conn, def_config, tmp_path) + create_functions(conn, def_config, sql_tmp_path) assert temp_db_cursor.scalar('SELECT test()') == 43 @pytest.mark.parametrize("dbg,ret", ((True, 43), (False, 22))) -def test_create_functions_with_template(temp_db_cursor, conn, def_config, tmp_path, dbg, ret): - sqlfile = tmp_path / 'functions.sql' +def test_create_functions_with_template(temp_db_cursor, conn, def_config, sql_tmp_path, dbg, ret): + sqlfile = sql_tmp_path / 'functions.sql' sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER AS $$ BEGIN @@ -42,6 +47,6 @@ def test_create_functions_with_template(temp_db_cursor, conn, def_config, tmp_pa $$ LANGUAGE plpgsql IMMUTABLE; """) - create_functions(conn, def_config, tmp_path, enable_debug=dbg) + create_functions(conn, def_config, sql_tmp_path, enable_debug=dbg) assert temp_db_cursor.scalar('SELECT test()') == ret -- 2.39.5