" wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz")
endif()
- set(CUSTOMSCRIPTS
- check_import_finished.php
- country_languages.php
- export.php
- query.php
- setup.php
- update.php
- warm.php
- )
-
- foreach (script_source ${CUSTOMSCRIPTS})
- configure_file(${PROJECT_SOURCE_DIR}/cmake/script.tmpl
- ${PROJECT_BINARY_DIR}/utils/${script_source})
- endforeach()
-
configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl
${PROJECT_BINARY_DIR}/nominatim)
endif()
+++ /dev/null
-#!@PHP_BIN@ -Cq
-<?php
-require('@CMAKE_SOURCE_DIR@/lib-php/dotenv_loader.php');
-
-@define('CONST_Default_ModulePath', '@CMAKE_BINARY_DIR@/module');
-@define('CONST_Default_Osm2pgsql', '@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql');
-@define('CONST_DataDir', '@CMAKE_SOURCE_DIR@/data');
-@define('CONST_SqlDir', '@CMAKE_SOURCE_DIR@/lib-sql');
-@define('CONST_ConfigDir', '@CMAKE_SOURCE_DIR@/settings');
-
-loadDotEnv();
-$_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim';
-
-require_once('@CMAKE_SOURCE_DIR@/lib-php/admin/@script_source@');
+++ /dev/null
-<?php
-@define('CONST_LibDir', dirname(dirname(__FILE__)));
-
-require_once(CONST_LibDir.'/init-cmd.php');
-
-loadSettings(getcwd());
-
-(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
- ->addParams('admin', '--check-database')
- ->run();
+++ /dev/null
-<?php
-@define('CONST_LibDir', dirname(dirname(__FILE__)));
-
-require_once(CONST_LibDir.'/init-cmd.php');
-require_once(CONST_LibDir.'/setup/SetupClass.php');
-require_once(CONST_LibDir.'/setup_functions.php');
-ini_set('memory_limit', '800M');
-
-use Nominatim\Setup\SetupFunctions as SetupFunctions;
-
-// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
-$aCMDOptions
-= array(
- 'Create and setup nominatim search system',
- array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
- array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
- array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
-
- array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
- array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
-
- array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
-
- array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
- array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
- array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
- array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
- array('reverse-only', '', 0, 1, 0, 0, 'bool', 'Do not create search tables and indexes'),
- array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
- array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
- array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
- array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
- array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
- array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
- array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
- array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
- array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
- array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
- array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
- array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
- array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
- array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
- array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
- array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
- array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'),
- array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
- array('setup-website', '', 0, 1, 0, 0, 'bool', 'Used to compile environment variables for the website'),
- array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
- );
-
-// $aCMDOptions passed to getCmdOpt by reference
-getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
-
-loadSettings($aCMDResult['project-dir'] ?? getcwd());
-setupHTTPProxy();
-
-$bDidSomething = false;
-
-$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
-
-// by default, use all but one processor, but never more than 15.
-$iInstances = max(1, $aCMDResult['threads'] ?? (min(16, getProcessorCount()) - 1));
-
-function run($oCmd)
-{
- global $iInstances;
- global $aCMDResult;
- $oCmd->addParams('--threads', $iInstances);
- if ($aCMDResult['ignore-errors'] ?? false) {
- $oCmd->addParams('--ignore-errors');
- }
- if ($aCMDResult['quiet'] ?? false) {
- $oCmd->addParams('--quiet');
- }
- if ($aCMDResult['verbose'] ?? false) {
- $oCmd->addParams('--verbose');
- }
- $oCmd->run(true);
-}
-
-
-//*******************************************************
-// Making some sanity check:
-// Check if osm-file is set and points to a valid file
-if ($aCMDResult['import-data'] || $aCMDResult['all']) {
- // to remain in /lib/setup_functions.php function
- checkInFile($aCMDResult['osm-file']);
-}
-
-// ******************************************************
-// instantiate Setup class
-$oSetup = new SetupFunctions($aCMDResult);
-
-// *******************************************************
-// go through complete process if 'all' is selected or start selected functions
-if ($aCMDResult['create-db'] || $aCMDResult['all']) {
- $bDidSomething = true;
- run((clone($oNominatimCmd))->addParams('transition', '--create-db'));
-}
-
-if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
- $bDidSomething = true;
- $oCmd = (clone($oNominatimCmd))->addParams('transition', '--setup-db');
-
- if ($aCMDResult['no-partitions'] ?? false) {
- $oCmd->addParams('--no-partitions');
- }
-
- run($oCmd);
-}
-
-if ($aCMDResult['import-data'] || $aCMDResult['all']) {
- $bDidSomething = true;
- $oCmd = (clone($oNominatimCmd))
- ->addParams('transition', '--import-data')
- ->addParams('--osm-file', $aCMDResult['osm-file']);
- if ($aCMDResult['drop'] ?? false) {
- $oCmd->addParams('--drop');
- }
-
- run($oCmd);
-}
-
-if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
- $bDidSomething = true;
- $oSetup->createSqlFunctions();
-}
-
-if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
- $bDidSomething = true;
- $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables');
-
- if ($aCMDResult['reverse-only'] ?? false) {
- $oCmd->addParams('--reverse-only');
- }
-
- run($oCmd);
-}
-
-if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
- $bDidSomething = true;
- run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables'));
-}
-
-if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
- $bDidSomething = true;
- $oSetup->createSqlFunctions(); // also create partition functions
-}
-
-if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
- $bDidSomething = true;
- // ignore errors!
- (clone($oNominatimCmd))->addParams('refresh', '--wiki-data')->run();
-}
-
-if ($aCMDResult['load-data'] || $aCMDResult['all']) {
- $bDidSomething = true;
- run((clone($oNominatimCmd))->addParams('transition', '--load-data'));
-}
-
-if ($aCMDResult['import-tiger-data']) {
- $bDidSomething = true;
- $sTigerPath = getSetting('TIGER_DATA_PATH', CONST_InstallDir.'/tiger');
- run((clone($oNominatimCmd))->addParams('transition', '--tiger-data', $sTigerPath));
-}
-
-if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
- $bDidSomething = true;
- $oSetup->calculatePostcodes($aCMDResult['all']);
-}
-
-if ($aCMDResult['index'] || $aCMDResult['all']) {
- $bDidSomething = true;
- $oCmd = (clone($oNominatimCmd))->addParams('transition', '--index');
- if ($aCMDResult['index-noanalyse'] ?? false) {
- $oCmd->addParams('--no-analyse');
- }
-
- run($oCmd);
-}
-
-if ($aCMDResult['drop']) {
- $bDidSomething = true;
- run((clone($oNominatimCmd))->addParams('freeze'));
-}
-
-if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
- $bDidSomething = true;
-
- $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-search-indices');
-
- if ($aCMDResult['drop'] ?? false) {
- $oCmd->addParams('--drop');
- }
-
- run($oCmd);
-}
-
-if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
- $bDidSomething = true;
- run(clone($oNominatimCmd))->addParams('transition', '--create-country-names');
-}
-
-if ($aCMDResult['setup-website'] || $aCMDResult['all']) {
- $bDidSomething = true;
- run((clone($oNominatimCmd))->addParams('refresh', '--website'));
-}
-
-// ******************************************************
-// If we did something, repeat the warnings
-if (!$bDidSomething) {
- showUsage($aCMDOptions, true);
-} else {
- echo "Summary of warnings:\n\n";
- repeatWarnings();
- echo "\n";
- info('Setup finished.');
-}
+++ /dev/null
-
-<?php
-@define('CONST_LibDir', dirname(dirname(__FILE__)));
-
-require_once(CONST_LibDir.'/init-cmd.php');
-
-loadSettings(getcwd());
-
-(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
- ->addParams('special-phrases', '--import-from-wiki')
- ->run();
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/setup_functions.php');
-require_once(CONST_LibDir.'/setup/SetupClass.php');
ini_set('memory_limit', '800M');
-use Nominatim\Setup\SetupFunctions as SetupFunctions;
-
// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
$aCMDOptions
= array(
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
- array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
- array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
- array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
- array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
- array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
- array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
-
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
- array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
- array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
- array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
-
- array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
- array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'),
- array('recompute-importance', '', 0, 1, 0, 0, 'bool', 'Recompute place importances'),
-
array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
);
loadSettings($aCMDResult['project-dir'] ?? getcwd());
setupHTTPProxy();
-if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
-if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
-
date_default_timezone_set('Etc/UTC');
$oDB = new Nominatim\DB();
);
}
-$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
-
-function run($oCmd)
-{
- global $aCMDResult;
- if ($aCMDResult['quiet'] ?? false) {
- $oCmd->addParams('--quiet');
- }
- if ($aCMDResult['verbose'] ?? false) {
- $oCmd->addParams('--verbose');
- }
- $oCmd->run(true);
-}
-
-
-if ($aResult['init-updates']) {
- $oCmd = (clone($oNominatimCmd))->addParams('replication', '--init');
-
- if ($aResult['no-update-functions']) {
- $oCmd->addParams('--no-update-functions');
- }
-
- run($oCmd);
-}
-
-if ($aResult['check-for-updates']) {
- exit((clone($oNominatimCmd))->addParams('replication', '--check-for-updates')->run());
-}
-
if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
// import diffs and files directly (e.g. from osmosis --rri)
$sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
// Don't update the import status - we don't know what this file contains
}
-if ($aResult['calculate-postcodes']) {
- run((clone($oNominatimCmd))->addParams('refresh', '--postcodes'));
-}
-
$sTemporaryFile = CONST_InstallDir.'/osmosischange.osc';
$bHaveDiff = false;
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
fail("osm2pgsql exited with error level $iRet\n");
}
}
-
-if ($aResult['recompute-word-counts']) {
- run((clone($oNominatimCmd))->addParams('refresh', '--word-counts'));
-}
-
-if ($aResult['index']) {
- run((clone $oNominatimCmd)
- ->addParams('index', '--minrank', $aResult['index-rank'])
- ->addParams('--threads', $aResult['index-instances']));
-}
-
-if ($aResult['update-address-levels']) {
- run((clone($oNominatimCmd))->addParams('refresh', '--address-levels'));
-}
-
-if ($aResult['recompute-importance']) {
- run((clone($oNominatimCmd))->addParams('refresh', '--importance'));
-}
-
-if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
- $oCmd = (clone($oNominatimCmd))
- ->addParams('replication')
- ->addParams('--threads', $aResult['index-instances']);
-
- if (!$aResult['import-osmosis-all']) {
- $oCmd->addParams('--once');
- }
-
- if ($aResult['no-index']) {
- $oCmd->addParams('--no-index');
- }
-
- run($oCmd);
-}
}
-function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreErrors = false)
-{
- // Convert database DSN to psql parameters
- $aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
- if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
-
- $oCmd = new \Nominatim\Shell('psql');
- $oCmd->addParams('--port', $aDSNInfo['port']);
- $oCmd->addParams('--dbname', $aDSNInfo['database']);
- if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
- $oCmd->addParams('--host', $aDSNInfo['hostspec']);
- }
- if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
- $oCmd->addParams('--username', $aDSNInfo['username']);
- }
- if (isset($aDSNInfo['password'])) {
- $oCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
- }
- if (!$bVerbose) {
- $oCmd->addParams('--quiet');
- }
- if ($bfatal && !$bIgnoreErrors) {
- $oCmd->addParams('-v', 'ON_ERROR_STOP=1');
- }
-
- $aDescriptors = array(
- 0 => array('pipe', 'r'),
- 1 => STDOUT,
- 2 => STDERR
- );
- $ahPipes = null;
- $hProcess = @proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
- if (!is_resource($hProcess)) {
- fail('unable to start pgsql');
- }
-
- if (!$bVerbose) {
- fwrite($ahPipes[0], 'set client_min_messages to WARNING;');
- }
-
- while (strlen($sScript)) {
- $iWritten = fwrite($ahPipes[0], $sScript);
- if ($iWritten <= 0) break;
- $sScript = substr($sScript, $iWritten);
- }
- fclose($ahPipes[0]);
- $iReturn = proc_close($hProcess);
- if ($bfatal && $iReturn > 0) {
- fail("pgsql returned with error code ($iReturn)");
- }
-}
-
function setupHTTPProxy()
{
if (!getSettingBool('HTTP_PROXY')) {
+++ /dev/null
-<?php
-
-namespace Nominatim\Setup;
-
-require_once(CONST_LibDir.'/Shell.php');
-
-class SetupFunctions
-{
- protected $iInstances;
- protected $aDSNInfo;
- protected $bQuiet;
- protected $bVerbose;
- protected $sIgnoreErrors;
- protected $bEnableDiffUpdates;
- protected $bEnableDebugStatements;
- protected $bDrop;
- protected $oDB = null;
- protected $oNominatimCmd;
-
- public function __construct(array $aCMDResult)
- {
- // by default, use all but one processor, but never more than 15.
- $this->iInstances = isset($aCMDResult['threads'])
- ? $aCMDResult['threads']
- : (min(16, getProcessorCount()) - 1);
-
- if ($this->iInstances < 1) {
- $this->iInstances = 1;
- warn('resetting threads to '.$this->iInstances);
- }
-
- // parse database string
- $this->aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
- if (!isset($this->aDSNInfo['port'])) {
- $this->aDSNInfo['port'] = 5432;
- }
-
- // setting member variables based on command line options stored in $aCMDResult
- $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet'];
- $this->bVerbose = $aCMDResult['verbose'];
-
- //setting default values which are not set by the update.php array
- if (isset($aCMDResult['ignore-errors'])) {
- $this->sIgnoreErrors = $aCMDResult['ignore-errors'];
- } else {
- $this->sIgnoreErrors = false;
- }
- if (isset($aCMDResult['enable-debug-statements'])) {
- $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements'];
- } else {
- $this->bEnableDebugStatements = false;
- }
- if (isset($aCMDResult['enable-diff-updates'])) {
- $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
- } else {
- $this->bEnableDiffUpdates = false;
- }
-
- $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop'];
-
- $this->oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
- if ($this->bQuiet) {
- $this->oNominatimCmd->addParams('--quiet');
- }
- if ($this->bVerbose) {
- $this->oNominatimCmd->addParams('--verbose');
- }
- }
-
- public function calculatePostcodes($bCMDResultAll)
- {
- info('Calculate Postcodes');
- $this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql');
-
- $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz';
- if (file_exists($sPostcodeFilename)) {
- $this->pgsqlRunScriptFile($sPostcodeFilename);
- } else {
- warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
- }
-
- $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz';
- if (file_exists($sPostcodeFilename)) {
- $this->pgsqlRunScriptFile($sPostcodeFilename);
- } else {
- warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
- }
-
-
- $this->db()->exec('TRUNCATE location_postcode');
-
- $sSQL = 'INSERT INTO location_postcode';
- $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
- $sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
- $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,";
- $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))';
- $sSQL .= ' FROM placex';
- $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
- $sSQL .= ' AND geometry IS NOT null';
- $sSQL .= ' GROUP BY country_code, pc';
- $this->db()->exec($sSQL);
-
- // only add postcodes that are not yet available in OSM
- $sSQL = 'INSERT INTO location_postcode';
- $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
- $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
- $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)';
- $sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
- $sSQL .= ' (SELECT postcode FROM location_postcode';
- $sSQL .= " WHERE country_code = 'us')";
- $this->db()->exec($sSQL);
-
- // add missing postcodes for GB (if available)
- $sSQL = 'INSERT INTO location_postcode';
- $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
- $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
- $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
- $sSQL .= ' (SELECT postcode FROM location_postcode';
- $sSQL .= " WHERE country_code = 'gb')";
- $this->db()->exec($sSQL);
-
- if (!$bCMDResultAll) {
- $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
- $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
- $this->db()->exec($sSQL);
- }
-
- $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
- $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
- $this->db()->exec($sSQL);
- }
-
- /**
- * Return the connection to the database.
- *
- * @return Database object.
- *
- * Creates a new connection if none exists yet. Otherwise reuses the
- * already established connection.
- */
- private function db()
- {
- if (is_null($this->oDB)) {
- $this->oDB = new \Nominatim\DB();
- $this->oDB->connect();
- }
-
- return $this->oDB;
- }
-
- private function pgsqlRunScript($sScript, $bfatal = true)
- {
- runSQLScript(
- $sScript,
- $bfatal,
- $this->bVerbose,
- $this->sIgnoreErrors
- );
- }
-
- public function createSqlFunctions()
- {
- $oCmd = (clone($this->oNominatimCmd))
- ->addParams('refresh', '--functions');
-
- if (!$this->bEnableDiffUpdates) {
- $oCmd->addParams('--no-diff-updates');
- }
-
- if ($this->bEnableDebugStatements) {
- $oCmd->addParams('--enable-debug-statements');
- }
-
- $oCmd->run(!$this->sIgnoreErrors);
- }
-
- private function pgsqlRunScriptFile($sFilename)
- {
- if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
-
- $oCmd = (new \Nominatim\Shell('psql'))
- ->addParams('--port', $this->aDSNInfo['port'])
- ->addParams('--dbname', $this->aDSNInfo['database']);
-
- if (!$this->bVerbose) {
- $oCmd->addParams('--quiet');
- }
- if (isset($this->aDSNInfo['hostspec'])) {
- $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
- }
- if (isset($this->aDSNInfo['username'])) {
- $oCmd->addParams('--username', $this->aDSNInfo['username']);
- }
- if (isset($this->aDSNInfo['password'])) {
- $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
- }
- $ahGzipPipes = null;
- if (preg_match('/\\.gz$/', $sFilename)) {
- $aDescriptors = array(
- 0 => array('pipe', 'r'),
- 1 => array('pipe', 'w'),
- 2 => array('file', '/dev/null', 'a')
- );
- $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename);
-
- $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes);
- if (!is_resource($hGzipProcess)) fail('unable to start zcat');
- $aReadPipe = $ahGzipPipes[1];
- fclose($ahGzipPipes[0]);
- } else {
- $oCmd->addParams('--file', $sFilename);
- $aReadPipe = array('pipe', 'r');
- }
- $aDescriptors = array(
- 0 => $aReadPipe,
- 1 => array('pipe', 'w'),
- 2 => array('file', '/dev/null', 'a')
- );
- $ahPipes = null;
-
- $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
- if (!is_resource($hProcess)) fail('unable to start pgsql');
- // TODO: error checking
- while (!feof($ahPipes[1])) {
- echo fread($ahPipes[1], 4096);
- }
- fclose($ahPipes[1]);
- $iReturn = proc_close($hProcess);
- if ($iReturn > 0) {
- fail("pgsql returned with error code ($iReturn)");
- }
- if ($ahGzipPipes) {
- fclose($ahGzipPipes[1]);
- proc_close($hGzipProcess);
- }
- }
-
- private function replaceSqlPatterns($sSql)
- {
- $sSql = str_replace('{www-user}', getSetting('DATABASE_WEBUSER'), $sSql);
-
- $aPatterns = array(
- '{ts:address-data}' => getSetting('TABLESPACE_ADDRESS_DATA'),
- '{ts:address-index}' => getSetting('TABLESPACE_ADDRESS_INDEX'),
- '{ts:search-data}' => getSetting('TABLESPACE_SEARCH_DATA'),
- '{ts:search-index}' => getSetting('TABLESPACE_SEARCH_INDEX'),
- '{ts:aux-data}' => getSetting('TABLESPACE_AUX_DATA'),
- '{ts:aux-index}' => getSetting('TABLESPACE_AUX_INDEX')
- );
-
- foreach ($aPatterns as $sPattern => $sTablespace) {
- if ($sTablespace) {
- $sSql = str_replace($sPattern, 'TABLESPACE "'.$sTablespace.'"', $sSql);
- } else {
- $sSql = str_replace($sPattern, '', $sSql);
- }
- }
-
- return $sSql;
- }
-}
<?php
-function checkInFile($sOSMFile)
-{
- if (!isset($sOSMFile)) {
- fail('missing --osm-file for data import');
- }
-
- if (!file_exists($sOSMFile)) {
- fail('the path supplied to --osm-file does not exist');
- }
-
- if (!is_readable($sOSMFile)) {
- fail('osm-file "' . $aCMDResult['osm-file'] . '" not readable');
- }
-}
-
function getOsm2pgsqlBinary()
{
$sBinary = getSetting('OSM2PGSQL_BINARY');
else:
parser.parser.epilog = 'php-cgi not found. Query commands not available.'
- parser.add_subcommand('transition', clicmd.AdminTransition)
-
return parser
from nominatim.clicmd.refresh import UpdateRefresh
from nominatim.clicmd.admin import AdminFuncs
from nominatim.clicmd.freeze import SetupFreeze
-from nominatim.clicmd.transition import AdminTransition
from nominatim.clicmd.special_phrases import ImportSpecialPhrases
+++ /dev/null
-"""
-Implementation of the 'transition' subcommand.
-
-This subcommand provides standins for functions that were available
-through the PHP scripts but are now no longer directly accessible.
-This module will be removed as soon as the transition phase is over.
-"""
-import logging
-from pathlib import Path
-
-from nominatim.db.connection import connect
-from nominatim.db import status
-from nominatim.errors import UsageError
-
-# Do not repeat documentation of subcommand classes.
-# pylint: disable=C0111
-# Using non-top-level imports to avoid eventually unused imports.
-# pylint: disable=E0012,C0415
-
-LOG = logging.getLogger()
-
-class AdminTransition:
- """\
- Internal functions for code transition. Do not use.
- """
-
- @staticmethod
- def add_args(parser):
- group = parser.add_argument_group('Sub-functions')
- group.add_argument('--create-db', action='store_true',
- help='Create nominatim db')
- group.add_argument('--setup-db', action='store_true',
- help='Build a blank nominatim db')
- group.add_argument('--import-data', action='store_true',
- help='Import a osm file')
- group.add_argument('--load-data', action='store_true',
- help='Copy data to live tables from import table')
- group.add_argument('--create-tables', action='store_true',
- help='Create main tables')
- group.add_argument('--create-partition-tables', action='store_true',
- help='Create required partition tables')
- group.add_argument('--index', action='store_true',
- help='Index the data')
- group.add_argument('--create-search-indices', action='store_true',
- help='Create additional indices required for search and update')
- group.add_argument('--create-country-names', action='store_true',
- help='Create search index for default country names.')
- group = parser.add_argument_group('Options')
- group.add_argument('--no-partitions', action='store_true',
- help='Do not partition search indices')
- group.add_argument('--osm-file', metavar='FILE',
- help='File to import')
- group.add_argument('--drop', action='store_true',
- help='Drop tables needed for updates, making the database readonly')
- group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
- help='Size of cache to be used by osm2pgsql (in MB)')
- group.add_argument('--no-analyse', action='store_true',
- help='Do not perform analyse operations during index')
- group.add_argument('--ignore-errors', action='store_true',
- help="Ignore certain erros on import.")
- group.add_argument('--reverse-only', action='store_true',
- help='Do not create search tables and indexes')
- group.add_argument('--tiger-data', metavar='FILE',
- help='File to import')
-
- @staticmethod
- def run(args): # pylint: disable=too-many-statements
- from ..tools import database_import, tiger_data
- from ..tools import refresh
-
- if args.create_db:
- LOG.warning('Create DB')
- database_import.create_db(args.config.get_libpq_dsn())
-
- if args.setup_db:
- LOG.warning('Setup DB')
-
- with connect(args.config.get_libpq_dsn()) as conn:
- database_import.setup_extensions(conn)
- database_import.install_module(args.module_dir, args.project_dir,
- args.config.DATABASE_MODULE_PATH,
- conn=conn)
-
- database_import.import_base_data(args.config.get_libpq_dsn(),
- args.data_dir, args.no_partitions)
-
- if args.import_data:
- LOG.warning('Import data')
- if not args.osm_file:
- raise UsageError('Missing required --osm-file argument')
- database_import.import_osm_data(Path(args.osm_file),
- args.osm2pgsql_options(0, 1),
- drop=args.drop,
- ignore_errors=args.ignore_errors)
-
- if args.create_tables:
- LOG.warning('Create Tables')
- with connect(args.config.get_libpq_dsn()) as conn:
- database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only)
- refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
- refresh.create_functions(conn, args.config, args.sqllib_dir,
- enable_diff_updates=False)
- database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
-
- if args.create_partition_tables:
- LOG.warning('Create Partition Tables')
- with connect(args.config.get_libpq_dsn()) as conn:
- database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
-
- if args.load_data:
- LOG.warning('Load data')
- with connect(args.config.get_libpq_dsn()) as conn:
- database_import.truncate_data_tables(conn, args.config.MAX_WORD_FREQUENCY)
- database_import.load_data(args.config.get_libpq_dsn(),
- args.data_dir,
- args.threads or 1)
-
- with connect(args.config.get_libpq_dsn()) as conn:
- try:
- status.set_status(conn, status.compute_database_date(conn))
- except Exception as exc: # pylint: disable=broad-except
- LOG.error('Cannot determine date of database: %s', exc)
-
- if args.index:
- LOG.warning('Indexing')
- from ..indexer.indexer import Indexer
- indexer = Indexer(args.config.get_libpq_dsn(), args.threads or 1)
- indexer.index_full()
-
- if args.create_search_indices:
- LOG.warning('Create Search indices')
- with connect(args.config.get_libpq_dsn()) as conn:
- database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop)
-
- if args.tiger_data:
- LOG.warning('Tiger data')
- tiger_data.add_tiger_data(args.config.get_libpq_dsn(),
- args.tiger_data,
- args.threads or 1,
- args.config,
- args.sqllib_dir)
-
- if args.create_country_names:
- LOG.warning('Create search index for default country names.')
- with connect(args.config.get_libpq_dsn()) as conn:
- database_import.create_country_names(conn, args.config)
projects / nominatim.git / commitdiff