X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/dae2761137c5ee95daf495e5cdf459fd8c7c17b4..1889643eca5754664da106665f3826960858eccf:/lib/setup/SetupClass.php diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index 1dd30c81..16c48729 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -3,6 +3,7 @@ namespace Nominatim\Setup; require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php'); +require_once(CONST_BasePath.'/lib/Shell.php'); class SetupFunctions { @@ -10,11 +11,13 @@ class SetupFunctions protected $iInstances; protected $sModulePath; protected $aDSNInfo; + protected $bQuiet; protected $bVerbose; protected $sIgnoreErrors; protected $bEnableDiffUpdates; protected $bEnableDebugStatements; protected $bNoPartitions; + protected $bDrop; protected $oDB = null; public function __construct(array $aCMDResult) @@ -49,6 +52,7 @@ class SetupFunctions } // setting member variables based on command line options stored in $aCMDResult + $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet']; $this->bVerbose = $aCMDResult['verbose']; //setting default values which are not set by the update.php array @@ -72,6 +76,8 @@ class SetupFunctions } else { $this->bEnableDiffUpdates = false; } + + $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop']; } public function createDB() @@ -79,23 +85,27 @@ class SetupFunctions info('Create DB'); $oDB = new \Nominatim\DB; - if ($oDB->databaseExists()) { + if ($oDB->checkConnection()) { fail('database already exists ('.CONST_Database_DSN.')'); } - $sCreateDBCmd = 'createdb -E UTF-8' - .' -p '.escapeshellarg($this->aDSNInfo['port']) - .' '.escapeshellarg($this->aDSNInfo['database']); + $oCmd = (new \Nominatim\Shell('createdb')) + ->addParams('-E', 'UTF-8') + ->addParams('-p', $this->aDSNInfo['port']); + if (isset($this->aDSNInfo['username'])) { - $sCreateDBCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oCmd->addParams('-U', $this->aDSNInfo['username']); + } + if (isset($this->aDSNInfo['password'])) { + $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } - if (isset($this->aDSNInfo['hostspec'])) { - $sCreateDBCmd .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']); + $oCmd->addParams('-h', $this->aDSNInfo['hostspec']); } + $oCmd->addParams($this->aDSNInfo['database']); - $result = $this->runWithPgEnv($sCreateDBCmd); - if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); + $result = $oCmd->run(); + if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd()); } public function connect() @@ -169,43 +179,58 @@ class SetupFunctions { info('Import data'); - $osm2pgsql = CONST_Osm2pgsql_Binary; - if (!file_exists($osm2pgsql)) { + if (!file_exists(CONST_Osm2pgsql_Binary)) { echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '$osm2pgsql'"); + fail("osm2pgsql not found in '".CONST_Osm2pgsql_Binary."'"); } - $osm2pgsql .= ' -S '.escapeshellarg(CONST_Import_Style); + $oCmd = new \Nominatim\Shell(CONST_Osm2pgsql_Binary); + $oCmd->addParams('--style', CONST_Import_Style); if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes '.escapeshellarg(CONST_Osm2pgsql_Flatnode_File); - } - - if (CONST_Tablespace_Osm2pgsql_Data) - $osm2pgsql .= ' --tablespace-slim-data '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Data); - if (CONST_Tablespace_Osm2pgsql_Index) - $osm2pgsql .= ' --tablespace-slim-index '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Index); - if (CONST_Tablespace_Place_Data) - $osm2pgsql .= ' --tablespace-main-data '.escapeshellarg(CONST_Tablespace_Place_Data); - if (CONST_Tablespace_Place_Index) - $osm2pgsql .= ' --tablespace-main-index '.escapeshellarg(CONST_Tablespace_Place_Index); - $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.escapeshellarg($this->iCacheMemory); - $osm2pgsql .= ' -P '.escapeshellarg($this->aDSNInfo['port']); + $oCmd->addParams('--flat-nodes', CONST_Osm2pgsql_Flatnode_File); + } + if (CONST_Tablespace_Osm2pgsql_Data) { + $oCmd->addParams('--tablespace-slim-data', CONST_Tablespace_Osm2pgsql_Data); + } + if (CONST_Tablespace_Osm2pgsql_Index) { + $oCmd->addParams('--tablespace-slim-index', CONST_Tablespace_Osm2pgsql_Index); + } + if (CONST_Tablespace_Place_Data) { + $oCmd->addParams('--tablespace-main-data', CONST_Tablespace_Place_Data); + } + if (CONST_Tablespace_Place_Index) { + $oCmd->addParams('--tablespace-main-index', CONST_Tablespace_Place_Index); + } + $oCmd->addParams('--latlong', '--slim', '--create'); + $oCmd->addParams('--output', 'gazetteer'); + $oCmd->addParams('--hstore'); + $oCmd->addParams('--number-processes', 1); + $oCmd->addParams('--cache', $this->iCacheMemory); + $oCmd->addParams('--port', $this->aDSNInfo['port']); + if (isset($this->aDSNInfo['username'])) { - $osm2pgsql .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oCmd->addParams('--username', $this->aDSNInfo['username']); + } + if (isset($this->aDSNInfo['password'])) { + $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } if (isset($this->aDSNInfo['hostspec'])) { - $osm2pgsql .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']); + $oCmd->addParams('--host', $this->aDSNInfo['hostspec']); } - $osm2pgsql .= ' -d '.escapeshellarg($this->aDSNInfo['database']).' '.escapeshellarg($sOSMFile); - - $this->runWithPgEnv($osm2pgsql); + $oCmd->addParams('--database', $this->aDSNInfo['database']); + $oCmd->addParams($sOSMFile); + $oCmd->run(); if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) { fail('No Data'); } + + if ($this->bDrop) { + $this->dropTable('planet_osm_nodes'); + $this->removeFlatnodeFile(); + } } public function createFunctions() @@ -235,6 +260,16 @@ class SetupFunctions $oAlParser->createTable($this->oDB, 'address_levels'); } + public function createTableTriggers() + { + info('Create Tables'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/table-triggers.sql'); + $sTemplate = $this->replaceSqlPatterns($sTemplate); + + $this->pgsqlRunScript($sTemplate, false); + } + public function createPartitionTables() { info('Create Partition Tables'); @@ -397,8 +432,10 @@ class SetupFunctions $aFilenames = glob(CONST_Tiger_Data_Path.'/*.sql'); info('Found '.count($aFilenames).' SQL files in path '.CONST_Tiger_Data_Path); - if (empty($aFilenames)) return; - + if (empty($aFilenames)) { + warn('Tiger data import selected but no files found in path '.CONST_Tiger_Data_Path); + return; + } $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); @@ -507,34 +544,48 @@ class SetupFunctions public function index($bIndexNoanalyse) { - $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i' - .' -d '.escapeshellarg($this->aDSNInfo['database']) - .' -P '.escapeshellarg($this->aDSNInfo['port']) - .' -t '.escapeshellarg($this->iInstances.$sOutputFile); + $oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py')) + ->addParams('--database', $this->aDSNInfo['database']) + ->addParams('--port', $this->aDSNInfo['port']) + ->addParams('--threads', $this->iInstances); + + if (!$this->bQuiet) { + $oBaseCmd->addParams('-v'); + } + if ($this->bVerbose) { + $oBaseCmd->addParams('-v'); + } if (isset($this->aDSNInfo['hostspec'])) { - $sBaseCmd .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']); + $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']); } if (isset($this->aDSNInfo['username'])) { - $sBaseCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oBaseCmd->addParams('--user', $this->aDSNInfo['username']); + } + if (isset($this->aDSNInfo['password'])) { + $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } info('Index ranks 0 - 4'); - $iStatus = $this->runWithPgEnv($sBaseCmd.' -R 4'); + $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4); + echo $oCmd->escapedCmd(); + + $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index ranks 5 - 25'); - $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 5 -R 25'); + $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25); + $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index ranks 26 - 30'); - $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 26'); + $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26); + $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } @@ -548,7 +599,19 @@ class SetupFunctions { info('Create Search indices'); + $sSQL = 'SELECT relname FROM pg_class, pg_index '; + $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid'; + $aInvalidIndices = $this->oDB->getCol($sSQL); + + foreach ($aInvalidIndices as $sIndexName) { + info("Cleaning up invalid index $sIndexName"); + $this->oDB->exec("DROP INDEX $sIndexName;"); + } + $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); + if (!$this->bDrop) { + $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_updates.src.sql'); + } if (!$this->dbReverseOnly()) { $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql'); } @@ -612,7 +675,7 @@ class SetupFunctions ); $aDropTables = array(); - $aHaveTables = $this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"); + $aHaveTables = $this->oDB->getListOfTables(); foreach ($aHaveTables as $sTable) { $bFound = false; @@ -628,6 +691,50 @@ class SetupFunctions $this->dropTable($sDrop); } + $this->removeFlatnodeFile(); + } + + /** + * Setup settings-frontend.php in the build/website directory + * + * @return null + */ + public function setupWebsite() + { + $rOutputFile = fopen(CONST_InstallPath.'/website/settings-frontend.php', 'w'); + + fwrite($rOutputFile, "bVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; @@ -649,13 +756,21 @@ class SetupFunctions private function createSqlFunctions() { $sBasePath = CONST_BasePath.'/sql/functions/'; - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); + $sTemplate = file_get_contents($sBasePath.'utils.sql'); $sTemplate .= file_get_contents($sBasePath.'normalization.sql'); + $sTemplate .= file_get_contents($sBasePath.'ranking.sql'); $sTemplate .= file_get_contents($sBasePath.'importance.sql'); $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql'); $sTemplate .= file_get_contents($sBasePath.'interpolation.sql'); - $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql'); - $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql'); + if ($this->oDB->tableExists('place')) { + $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql'); + } + if ($this->oDB->tableExists('placex')) { + $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql'); + } + if ($this->oDB->tableExists('location_postcode')) { + $sTemplate .= file_get_contents($sBasePath.'postcode_triggers.sql'); + } $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); if ($this->bEnableDiffUpdates) { $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); @@ -701,21 +816,21 @@ class SetupFunctions { if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - $sCMD = 'psql' - .' -p '.escapeshellarg($this->aDSNInfo['port']) - .' -d '.escapeshellarg($this->aDSNInfo['database']); + $oCmd = (new \Nominatim\Shell('psql')) + ->addParams('--port', $this->aDSNInfo['port']) + ->addParams('--dbname', $this->aDSNInfo['database']); + if (!$this->bVerbose) { - $sCMD .= ' -q'; + $oCmd->addParams('--quiet'); } if (isset($this->aDSNInfo['hostspec'])) { - $sCMD .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']); + $oCmd->addParams('--host', $this->aDSNInfo['hostspec']); } if (isset($this->aDSNInfo['username'])) { - $sCMD .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oCmd->addParams('--username', $this->aDSNInfo['username']); } - $aProcEnv = null; if (isset($this->aDSNInfo['password'])) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } $ahGzipPipes = null; if (preg_match('/\\.gz$/', $sFilename)) { @@ -724,12 +839,14 @@ class SetupFunctions 1 => array('pipe', 'w'), 2 => array('file', '/dev/null', 'a') ); - $hGzipProcess = proc_open('zcat '.escapeshellarg($sFilename), $aDescriptors, $ahGzipPipes); + $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename); + + $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes); if (!is_resource($hGzipProcess)) fail('unable to start zcat'); $aReadPipe = $ahGzipPipes[1]; fclose($ahGzipPipes[0]); } else { - $sCMD .= ' -f '.escapeshellarg($sFilename); + $oCmd->addParams('--file', $sFilename); $aReadPipe = array('pipe', 'r'); } $aDescriptors = array( @@ -738,7 +855,8 @@ class SetupFunctions 2 => array('file', '/dev/null', 'a') ); $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); + + $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv); if (!is_resource($hProcess)) fail('unable to start pgsql'); // TODO: error checking while (!feof($ahPipes[1])) { @@ -779,21 +897,6 @@ class SetupFunctions return $sSql; } - private function runWithPgEnv($sCmd) - { - if ($this->bVerbose) { - echo "Execute: $sCmd\n"; - } - - $aProcEnv = null; - - if (isset($this->aDSNInfo['password'])) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - - return runWithEnv($sCmd, $aProcEnv); - } - /** * Drop table with the given name if it exists. * @@ -806,7 +909,7 @@ class SetupFunctions private function dropTable($sName) { if ($this->bVerbose) echo "Dropping table $sName\n"; - $this->oDB->exec('DROP TABLE IF EXISTS '.$sName.' CASCADE'); + $this->oDB->deleteTable($sName); } /**