From b9517c99ae6297c2ab908d7fa324efd2d1b2f43a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 9 Feb 2021 15:26:56 +0100 Subject: [PATCH] rename sql directory to lib-sql Also introduces a separate constant for the sql directory, so that it can be put separately from the rest of the data if required. --- cmake/script.tmpl | 1 + cmake/tool.tmpl | 1 + lib-php/lib.php | 1 + lib-php/setup/SetupClass.php | 18 +++++++++--------- {sql => lib-sql}/aux_tables.sql | 0 {sql => lib-sql}/functions/address_lookup.sql | 0 {sql => lib-sql}/functions/aux_property.sql | 0 {sql => lib-sql}/functions/importance.sql | 0 {sql => lib-sql}/functions/interpolation.sql | 0 {sql => lib-sql}/functions/normalization.sql | 0 {sql => lib-sql}/functions/place_triggers.sql | 0 {sql => lib-sql}/functions/placex_triggers.sql | 0 .../functions/postcode_triggers.sql | 0 {sql => lib-sql}/functions/ranking.sql | 0 {sql => lib-sql}/functions/utils.sql | 0 {sql => lib-sql}/indices.src.sql | 0 {sql => lib-sql}/indices_search.src.sql | 0 {sql => lib-sql}/indices_updates.src.sql | 0 {sql => lib-sql}/partition-functions.src.sql | 0 {sql => lib-sql}/partition-tables.src.sql | 0 {sql => lib-sql}/postcode_tables.sql | 0 {sql => lib-sql}/table-triggers.sql | 0 {sql => lib-sql}/tables.sql | 0 {sql => lib-sql}/tiger_import_finish.sql | 0 {sql => lib-sql}/tiger_import_start.sql | 0 {sql => lib-sql}/update-postcodes.sql | 0 {sql => lib-sql}/words.sql | 0 {sql => lib-sql}/words_from_search_name.sql | 0 nominatim/cli.py | 3 ++- nominatim/clicmd/refresh.py | 4 ++-- nominatim/clicmd/replication.py | 2 +- nominatim/tools/exec_utils.py | 1 + nominatim/tools/refresh.py | 12 +++++------- test/bdd/steps/nominatim_environment.py | 6 +++++- .../test_tools_refresh_create_functions.py | 2 +- 35 files changed, 29 insertions(+), 22 deletions(-) rename {sql => lib-sql}/aux_tables.sql (100%) rename {sql => lib-sql}/functions/address_lookup.sql (100%) rename {sql => lib-sql}/functions/aux_property.sql (100%) rename {sql => lib-sql}/functions/importance.sql (100%) rename {sql => lib-sql}/functions/interpolation.sql (100%) rename {sql => lib-sql}/functions/normalization.sql (100%) rename {sql => lib-sql}/functions/place_triggers.sql (100%) rename {sql => lib-sql}/functions/placex_triggers.sql (100%) rename {sql => lib-sql}/functions/postcode_triggers.sql (100%) rename {sql => lib-sql}/functions/ranking.sql (100%) rename {sql => lib-sql}/functions/utils.sql (100%) rename {sql => lib-sql}/indices.src.sql (100%) rename {sql => lib-sql}/indices_search.src.sql (100%) rename {sql => lib-sql}/indices_updates.src.sql (100%) rename {sql => lib-sql}/partition-functions.src.sql (100%) rename {sql => lib-sql}/partition-tables.src.sql (100%) rename {sql => lib-sql}/postcode_tables.sql (100%) rename {sql => lib-sql}/table-triggers.sql (100%) rename {sql => lib-sql}/tables.sql (100%) rename {sql => lib-sql}/tiger_import_finish.sql (100%) rename {sql => lib-sql}/tiger_import_start.sql (100%) rename {sql => lib-sql}/update-postcodes.sql (100%) rename {sql => lib-sql}/words.sql (100%) rename {sql => lib-sql}/words_from_search_name.sql (100%) diff --git a/cmake/script.tmpl b/cmake/script.tmpl index 5fdfe542..f0fa2118 100755 --- a/cmake/script.tmpl +++ b/cmake/script.tmpl @@ -6,6 +6,7 @@ require('@CMAKE_SOURCE_DIR@/lib-php/dotenv_loader.php'); @define('CONST_Default_Osm2pgsql', '@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql'); @define('CONST_BinDir', '@CMAKE_SOURCE_DIR@/utils'); @define('CONST_DataDir', '@CMAKE_SOURCE_DIR@'); +@define('CONST_SqlDir', '@CMAKE_SOURCE_DIR@/lib-sql'); loadDotEnv(); $_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim'; diff --git a/cmake/tool.tmpl b/cmake/tool.tmpl index 67d57ad7..76f80924 100755 --- a/cmake/tool.tmpl +++ b/cmake/tool.tmpl @@ -11,5 +11,6 @@ from nominatim import cli exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module', osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql', phplib_dir='@CMAKE_SOURCE_DIR@/lib-php', + sqllib_dir='@CMAKE_SOURCE_DIR@/lib-sql', data_dir='@CMAKE_SOURCE_DIR@', phpcgi_path='@PHPCGI_BIN@')) diff --git a/lib-php/lib.php b/lib-php/lib.php index a02fefd0..04b9bf86 100644 --- a/lib-php/lib.php +++ b/lib-php/lib.php @@ -8,6 +8,7 @@ function loadSettings($sProjectDir) // set of settings. defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']); defined('CONST_BinDir') or define('CONST_BinDir', $_SERVER['NOMINATIM_BINDIR']); + defined('CONST_SqlDir') or define('CONST_SqlDir', $_SERVER['NOMINATIM_SQLDIR']); defined('CONST_Default_ModulePath') or define('CONST_Default_ModulePath', $_SERVER['NOMINATIM_DATABASE_MODULE_SRC_PATH']); } diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php index 2a498287..80dcabb2 100755 --- a/lib-php/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -253,7 +253,7 @@ class SetupFunctions { info('Create Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tables.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -269,7 +269,7 @@ class SetupFunctions { info('Create Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/table-triggers.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -279,7 +279,7 @@ class SetupFunctions { info('Create Partition Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/partition-tables.src.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunPartitionScript($sTemplate); @@ -442,7 +442,7 @@ class SetupFunctions warn('Tiger data import selected but no files found in path '.$sTigerPath); return; } - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_start.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_start.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -496,7 +496,7 @@ class SetupFunctions } info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_finish.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_finish.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -505,7 +505,7 @@ class SetupFunctions public function calculatePostcodes($bCMDResultAll) { info('Calculate Postcodes'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/sql/postcode_tables.sql'); + $this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql'); $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz'; if (file_exists($sPostcodeFilename)) { @@ -621,12 +621,12 @@ class SetupFunctions $this->db()->exec("DROP INDEX $sIndexName;"); } - $sTemplate = file_get_contents(CONST_DataDir.'/sql/indices.src.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql'); if (!$this->bDrop) { - $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_updates.src.sql'); + $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql'); } if (!$this->dbReverseOnly()) { - $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_search.src.sql'); + $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql'); } $sTemplate = $this->replaceSqlPatterns($sTemplate); diff --git a/sql/aux_tables.sql b/lib-sql/aux_tables.sql similarity index 100% rename from sql/aux_tables.sql rename to lib-sql/aux_tables.sql diff --git a/sql/functions/address_lookup.sql b/lib-sql/functions/address_lookup.sql similarity index 100% rename from sql/functions/address_lookup.sql rename to lib-sql/functions/address_lookup.sql diff --git a/sql/functions/aux_property.sql b/lib-sql/functions/aux_property.sql similarity index 100% rename from sql/functions/aux_property.sql rename to lib-sql/functions/aux_property.sql diff --git a/sql/functions/importance.sql b/lib-sql/functions/importance.sql similarity index 100% rename from sql/functions/importance.sql rename to lib-sql/functions/importance.sql diff --git a/sql/functions/interpolation.sql b/lib-sql/functions/interpolation.sql similarity index 100% rename from sql/functions/interpolation.sql rename to lib-sql/functions/interpolation.sql diff --git a/sql/functions/normalization.sql b/lib-sql/functions/normalization.sql similarity index 100% rename from sql/functions/normalization.sql rename to lib-sql/functions/normalization.sql diff --git a/sql/functions/place_triggers.sql b/lib-sql/functions/place_triggers.sql similarity index 100% rename from sql/functions/place_triggers.sql rename to lib-sql/functions/place_triggers.sql diff --git a/sql/functions/placex_triggers.sql b/lib-sql/functions/placex_triggers.sql similarity index 100% rename from sql/functions/placex_triggers.sql rename to lib-sql/functions/placex_triggers.sql diff --git a/sql/functions/postcode_triggers.sql b/lib-sql/functions/postcode_triggers.sql similarity index 100% rename from sql/functions/postcode_triggers.sql rename to lib-sql/functions/postcode_triggers.sql diff --git a/sql/functions/ranking.sql b/lib-sql/functions/ranking.sql similarity index 100% rename from sql/functions/ranking.sql rename to lib-sql/functions/ranking.sql diff --git a/sql/functions/utils.sql b/lib-sql/functions/utils.sql similarity index 100% rename from sql/functions/utils.sql rename to lib-sql/functions/utils.sql diff --git a/sql/indices.src.sql b/lib-sql/indices.src.sql similarity index 100% rename from sql/indices.src.sql rename to lib-sql/indices.src.sql diff --git a/sql/indices_search.src.sql b/lib-sql/indices_search.src.sql similarity index 100% rename from sql/indices_search.src.sql rename to lib-sql/indices_search.src.sql diff --git a/sql/indices_updates.src.sql b/lib-sql/indices_updates.src.sql similarity index 100% rename from sql/indices_updates.src.sql rename to lib-sql/indices_updates.src.sql diff --git a/sql/partition-functions.src.sql b/lib-sql/partition-functions.src.sql similarity index 100% rename from sql/partition-functions.src.sql rename to lib-sql/partition-functions.src.sql diff --git a/sql/partition-tables.src.sql b/lib-sql/partition-tables.src.sql similarity index 100% rename from sql/partition-tables.src.sql rename to lib-sql/partition-tables.src.sql diff --git a/sql/postcode_tables.sql b/lib-sql/postcode_tables.sql similarity index 100% rename from sql/postcode_tables.sql rename to lib-sql/postcode_tables.sql diff --git a/sql/table-triggers.sql b/lib-sql/table-triggers.sql similarity index 100% rename from sql/table-triggers.sql rename to lib-sql/table-triggers.sql diff --git a/sql/tables.sql b/lib-sql/tables.sql similarity index 100% rename from sql/tables.sql rename to lib-sql/tables.sql diff --git a/sql/tiger_import_finish.sql b/lib-sql/tiger_import_finish.sql similarity index 100% rename from sql/tiger_import_finish.sql rename to lib-sql/tiger_import_finish.sql diff --git a/sql/tiger_import_start.sql b/lib-sql/tiger_import_start.sql similarity index 100% rename from sql/tiger_import_start.sql rename to lib-sql/tiger_import_start.sql diff --git a/sql/update-postcodes.sql b/lib-sql/update-postcodes.sql similarity index 100% rename from sql/update-postcodes.sql rename to lib-sql/update-postcodes.sql diff --git a/sql/words.sql b/lib-sql/words.sql similarity index 100% rename from sql/words.sql rename to lib-sql/words.sql diff --git a/sql/words_from_search_name.sql b/lib-sql/words_from_search_name.sql similarity index 100% rename from sql/words_from_search_name.sql rename to lib-sql/words_from_search_name.sql diff --git a/nominatim/cli.py b/nominatim/cli.py index 845b1451..784250e8 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -68,7 +68,8 @@ class CommandlineParser: self.parser.print_help() return 1 - for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'): + for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'sqllib_dir', + 'data_dir', 'phpcgi_path'): setattr(args, arg, Path(kwargs[arg])) args.project_dir = Path(args.project_dir).resolve() diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py index 4e2ad1db..baff9b6a 100644 --- a/nominatim/clicmd/refresh.py +++ b/nominatim/clicmd/refresh.py @@ -51,7 +51,7 @@ class UpdateRefresh: if args.postcodes: LOG.warning("Update postcodes centroid") conn = connect(args.config.get_libpq_dsn()) - refresh.update_postcodes(conn, args.data_dir) + refresh.update_postcodes(conn, args.sqllib_dir) conn.close() if args.word_counts: @@ -70,7 +70,7 @@ class UpdateRefresh: if args.functions: LOG.warning('Create functions') conn = connect(args.config.get_libpq_dsn()) - refresh.create_functions(conn, args.config, args.data_dir, + refresh.create_functions(conn, args.config, args.sqllib_dir, args.diffs, args.enable_debug_statements) conn.close() diff --git a/nominatim/clicmd/replication.py b/nominatim/clicmd/replication.py index 554dbc4f..4852c8b1 100644 --- a/nominatim/clicmd/replication.py +++ b/nominatim/clicmd/replication.py @@ -66,7 +66,7 @@ class UpdateReplication: replication.init_replication(conn, base_url=args.config.REPLICATION_URL) if args.update_functions: LOG.warning("Create functions") - refresh.create_functions(conn, args.config, args.data_dir, + refresh.create_functions(conn, args.config, args.sqllib_dir, True, False) conn.close() return 0 diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index 45853163..502fb4fe 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -26,6 +26,7 @@ def run_legacy_script(script, *args, nominatim_env=None, throw_on_fail=False): env = nominatim_env.config.get_os_env() env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir) env['NOMINATIM_BINDIR'] = str(nominatim_env.data_dir / 'utils') + env['NOMINATIM_SQLDIR'] = str(nominatim_env.sqllib_dir) env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = nominatim_env.module_dir if not env['NOMINATIM_OSM2PGSQL_BINARY']: env['NOMINATIM_OSM2PGSQL_BINARY'] = nominatim_env.osm2pgsql_path diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index 5fbb07f8..1fcb1577 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -8,17 +8,17 @@ from psycopg2.extras import execute_values from ..db.utils import execute_file -def update_postcodes(conn, datadir): +def update_postcodes(conn, sql_dir): """ Recalculate postcode centroids and add, remove and update entries in the location_postcode table. `conn` is an opne connection to the database. """ - execute_file(conn, datadir / 'sql' / 'update-postcodes.sql') + execute_file(conn, sql_dir / 'update-postcodes.sql') -def recompute_word_counts(conn, datadir): +def recompute_word_counts(conn, sql_dir): """ Compute the frequency of full-word search terms. """ - execute_file(conn, datadir / 'sql' / 'words_from_search_name.sql') + execute_file(conn, sql_dir / 'words_from_search_name.sql') def _add_address_level_rows_from_entry(rows, entry): @@ -153,12 +153,10 @@ def _get_partition_function_sql(conn, sql_dir): return replace_partition_string(sql, sorted(partitions)) -def create_functions(conn, config, data_dir, +def create_functions(conn, config, sql_dir, enable_diff_updates=True, enable_debug=False): """ (Re)create the PL/pgSQL functions. """ - sql_dir = data_dir / 'sql' - sql = _get_standard_function_sql(conn, config, sql_dir, enable_diff_updates, enable_debug) sql += _get_partition_function_sql(conn, sql_dir) diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 892d21e9..6dcaf64a 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -88,13 +88,17 @@ class NominatimEnvironment: self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full' self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes' self.test_env['NOMINATIM_DATADIR'] = self.src_dir + self.test_env['NOMINATIM_SQLDIR'] = self.src_dir / 'lib-sql' self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils' - self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module' + self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = self.build_dir / 'module' self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql' self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim' if self.server_module_path: self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path + else: + # avoid module being copied into the temporary environment + self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module' if self.website_dir is not None: self.website_dir.cleanup() diff --git a/test/python/test_tools_refresh_create_functions.py b/test/python/test_tools_refresh_create_functions.py index 4807e64f..d219d748 100644 --- a/test/python/test_tools_refresh_create_functions.py +++ b/test/python/test_tools_refresh_create_functions.py @@ -7,7 +7,7 @@ import pytest from nominatim.db.connection import connect from nominatim.tools.refresh import _get_standard_function_sql, _get_partition_function_sql -SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'sql').resolve() +SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-sql').resolve() @pytest.fixture def db(temp_db): -- 2.39.5