From a0dbeabed1a18e9bedb6cb2a6eed4c2b5fbf45cc Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Mon, 17 Sep 2018 10:28:00 +0200 Subject: [PATCH] move setupclass, move command line array, remove args from update array --- lib/{classes => setup}/SetupClass.php | 15 +++-- lib/setup_functions.php | 85 --------------------------- utils/setup.php | 40 ++++++++++++- utils/update.php | 40 ++++++++++++- 4 files changed, 86 insertions(+), 94 deletions(-) rename lib/{classes => setup}/SetupClass.php (98%) diff --git a/lib/classes/SetupClass.php b/lib/setup/SetupClass.php similarity index 98% rename from lib/classes/SetupClass.php rename to lib/setup/SetupClass.php index 09f37e51..f31c2353 100755 --- a/lib/classes/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -16,7 +16,7 @@ class SetupFunctions protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set - public function __construct($aCMDResult) + public function __construct($aCMDResult, $callingFunction = 'setup') { // by default, use all but one processor, but never more than 15. $this->iInstances = isset($aCMDResult['threads']) @@ -45,7 +45,6 @@ class SetupFunctions // setting member variables based on command line options stored in $aCMDResult $this->sVerbose = $aCMDResult['verbose']; - $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; //setting default values which are not set by the update.php array if (isset($aCMDResult['ignore-errors'])) { @@ -63,6 +62,14 @@ class SetupFunctions } else { $this->bNoPartitions = false; } + + // if class is instantiated by update.php, we have to set EnableDiffUpdates to true + // otherwise set to value provided comand line to setup.php + if ($callingFunction == 'update') { + $this->bEnableDiffUpdates = true; + } elseif ($callingFunction == 'setup') { + $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; + } } public function createDB() @@ -166,8 +173,8 @@ class SetupFunctions $this->pgsqlRunScript('update country_name set partition = 0'); } - // the following will be needed by create_functions later but - // is only defined in the subsequently called T + // the following will be needed by createFunctions later but + // is only defined in the subsequently called createTables // Create dummies here that will be overwritten by the proper // versions in create-tables. $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); diff --git a/lib/setup_functions.php b/lib/setup_functions.php index af718ed0..b1417678 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -34,90 +34,5 @@ function checkModulePresence() echo $oResult->userinfo . "\n\n"; $bResult = false; } - return $bResult; } - -// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) -// create and array -function createSetupArgvArray() -{ - $aCMDOptions - = array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - ); - return $aCMDOptions; -} - -function createUpdateArgvArray() -{ - $aCMDOptions - = array( - 'Import / update / index osm data', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), - array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), - array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), - array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), - array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), - array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), - - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), - - array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), - array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - - array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), - array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), - array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), - array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), - - array('index', '', 0, 1, 0, 0, 'bool', 'Index'), - array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), - array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), - - array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), - array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), - array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), - - array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'), - ); - return $aCMDOptions; -} diff --git a/utils/setup.php b/utils/setup.php index 96d66a16..5bc2a6f9 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -6,13 +6,49 @@ require_once(CONST_BasePath.'/lib/init-cmd.php'); // ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt // ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB() -require_once(CONST_BasePath.'/lib/classes/SetupClass.php'); +require_once(CONST_BasePath.'/lib/setup/SetupClass.php'); require_once(CONST_BasePath.'/lib/setup_functions.php'); ini_set('memory_limit', '800M'); use Nominatim\Setup\SetupFunctions as SetupFunctions; -$aCMDOptions = createSetupArgvArray(); +// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) +$aCMDOptions += array( + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), +); // $aCMDOptions passed to getCmdOpt by reference getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); diff --git a/utils/update.php b/utils/update.php index 0e12bb86..9175a617 100755 --- a/utils/update.php +++ b/utils/update.php @@ -4,13 +4,47 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once(CONST_BasePath.'/lib/init-cmd.php'); require_once(CONST_BasePath.'/lib/setup_functions.php'); -require_once(CONST_BasePath.'/lib/classes/SetupClass.php'); +require_once(CONST_BasePath.'/lib/setup/SetupClass.php'); ini_set('memory_limit', '800M'); use Nominatim\Setup\SetupFunctions as SetupFunctions; -$aCMDOptions = createUpdateArgvArray(); +// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) +$aCMDOptions += array( + 'Import / update / index osm data', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), + array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), + array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), + array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), + array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), + array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), + + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), + + array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), + array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + + array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), + array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), + array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), + array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), + + array('index', '', 0, 1, 0, 0, 'bool', 'Index'), + array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), + array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), + + array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), + array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), + array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), +); + getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true); if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1; @@ -72,7 +106,7 @@ if ($aResult['init-updates']) { if (!$aResult['no-update-functions']) { // instantiate setupClass to use the function therein - $cSetup = new SetupFunctions($aResult); + $cSetup = new SetupFunctions($aResult, 'update'); $cSetup->createFunctions(); } -- 2.39.5