X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/a06ceeef4ca8d42ceb104d8958514dae7b3a5bee..ddc2b4b80692eef15c4fcfab8de5840d8f9e2130:/lib/setup/SetupClass.php diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index aa1b291d..f16c9e60 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -3,6 +3,7 @@ namespace Nominatim\Setup; require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php'); +require_once(CONST_BasePath.'/lib/Shell.php'); class SetupFunctions { @@ -51,7 +52,7 @@ class SetupFunctions } // setting member variables based on command line options stored in $aCMDResult - $this->bQuiet = $aCMDResult['quiet']; + $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet']; $this->bVerbose = $aCMDResult['verbose']; //setting default values which are not set by the update.php array @@ -76,7 +77,7 @@ class SetupFunctions $this->bEnableDiffUpdates = false; } - $this->bDrop = $aCMDResult['drop']; + $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop']; } public function createDB() @@ -84,36 +85,34 @@ class SetupFunctions info('Create DB'); $oDB = new \Nominatim\DB; - if ($oDB->databaseExists()) { + if ($oDB->checkConnection()) { fail('database already exists ('.CONST_Database_DSN.')'); } - $sCreateDBCmd = 'createdb -E UTF-8' - .' -p '.escapeshellarg($this->aDSNInfo['port']) - .' '.escapeshellarg($this->aDSNInfo['database']); + $oCmd = (new \Nominatim\Shell('createdb')) + ->addParams('-E', 'UTF-8') + ->addParams('-p', $this->aDSNInfo['port']); + if (isset($this->aDSNInfo['username'])) { - $sCreateDBCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oCmd->addParams('-U', $this->aDSNInfo['username']); + } + if (isset($this->aDSNInfo['password'])) { + $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } - if (isset($this->aDSNInfo['hostspec'])) { - $sCreateDBCmd .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']); + $oCmd->addParams('-h', $this->aDSNInfo['hostspec']); } + $oCmd->addParams($this->aDSNInfo['database']); - $result = $this->runWithPgEnv($sCreateDBCmd); - if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); - } - - public function connect() - { - $this->oDB = new \Nominatim\DB(); - $this->oDB->connect(); + $result = $oCmd->run(); + if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd()); } public function setupDB() { info('Setup DB'); - $fPostgresVersion = $this->oDB->getPostgresVersion(); + $fPostgresVersion = $this->db()->getPostgresVersion(); echo 'Postgres version found: '.$fPostgresVersion."\n"; if ($fPostgresVersion < 9.03) { @@ -123,7 +122,7 @@ class SetupFunctions $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); - $fPostgisVersion = $this->oDB->getPostgisVersion(); + $fPostgisVersion = $this->db()->getPostgisVersion(); echo 'Postgis version found: '.$fPostgisVersion."\n"; if ($fPostgisVersion < 2.2) { @@ -131,7 +130,7 @@ class SetupFunctions exit(1); } - $i = $this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"); + $i = $this->db()->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"); if ($i == 0) { echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; echo "\n createuser ".CONST_Database_Web_User."\n\n"; @@ -174,41 +173,53 @@ class SetupFunctions { info('Import data'); - $osm2pgsql = CONST_Osm2pgsql_Binary; - if (!file_exists($osm2pgsql)) { + if (!file_exists(CONST_Osm2pgsql_Binary)) { echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '$osm2pgsql'"); + fail("osm2pgsql not found in '".CONST_Osm2pgsql_Binary."'"); } - $osm2pgsql .= ' -S '.escapeshellarg(CONST_Import_Style); + $oCmd = new \Nominatim\Shell(CONST_Osm2pgsql_Binary); + $oCmd->addParams('--style', CONST_Import_Style); if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes '.escapeshellarg(CONST_Osm2pgsql_Flatnode_File); - } - - if (CONST_Tablespace_Osm2pgsql_Data) - $osm2pgsql .= ' --tablespace-slim-data '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Data); - if (CONST_Tablespace_Osm2pgsql_Index) - $osm2pgsql .= ' --tablespace-slim-index '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Index); - if (CONST_Tablespace_Place_Data) - $osm2pgsql .= ' --tablespace-main-data '.escapeshellarg(CONST_Tablespace_Place_Data); - if (CONST_Tablespace_Place_Index) - $osm2pgsql .= ' --tablespace-main-index '.escapeshellarg(CONST_Tablespace_Place_Index); - $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.escapeshellarg($this->iCacheMemory); - $osm2pgsql .= ' -P '.escapeshellarg($this->aDSNInfo['port']); + $oCmd->addParams('--flat-nodes', CONST_Osm2pgsql_Flatnode_File); + } + if (CONST_Tablespace_Osm2pgsql_Data) { + $oCmd->addParams('--tablespace-slim-data', CONST_Tablespace_Osm2pgsql_Data); + } + if (CONST_Tablespace_Osm2pgsql_Index) { + $oCmd->addParams('--tablespace-slim-index', CONST_Tablespace_Osm2pgsql_Index); + } + if (CONST_Tablespace_Place_Data) { + $oCmd->addParams('--tablespace-main-data', CONST_Tablespace_Place_Data); + } + if (CONST_Tablespace_Place_Index) { + $oCmd->addParams('--tablespace-main-index', CONST_Tablespace_Place_Index); + } + $oCmd->addParams('--latlong', '--slim', '--create'); + $oCmd->addParams('--output', 'gazetteer'); + $oCmd->addParams('--hstore'); + $oCmd->addParams('--number-processes', 1); + $oCmd->addParams('--with-forward-dependencies', 'false'); + $oCmd->addParams('--log-progress', 'true'); + $oCmd->addParams('--cache', $this->iCacheMemory); + $oCmd->addParams('--port', $this->aDSNInfo['port']); + if (isset($this->aDSNInfo['username'])) { - $osm2pgsql .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oCmd->addParams('--username', $this->aDSNInfo['username']); + } + if (isset($this->aDSNInfo['password'])) { + $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } if (isset($this->aDSNInfo['hostspec'])) { - $osm2pgsql .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']); + $oCmd->addParams('--host', $this->aDSNInfo['hostspec']); } - $osm2pgsql .= ' -d '.escapeshellarg($this->aDSNInfo['database']).' '.escapeshellarg($sOSMFile); + $oCmd->addParams('--database', $this->aDSNInfo['database']); + $oCmd->addParams($sOSMFile); + $oCmd->run(); - $this->runWithPgEnv($osm2pgsql); - - if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) { + if (!$this->sIgnoreErrors && !$this->db()->getRow('select * from place limit 1')) { fail('No Data'); } @@ -242,7 +253,7 @@ class SetupFunctions } $oAlParser = new AddressLevelParser(CONST_Address_Level_Config); - $oAlParser->createTable($this->oDB, 'address_levels'); + $oAlParser->createTable($this->db(), 'address_levels'); } public function createTableTriggers() @@ -290,40 +301,42 @@ class SetupFunctions { info('Drop old Data'); - $this->oDB->exec('TRUNCATE word'); + $oDB = $this->db(); + + $oDB->exec('TRUNCATE word'); echo '.'; - $this->oDB->exec('TRUNCATE placex'); + $oDB->exec('TRUNCATE placex'); echo '.'; - $this->oDB->exec('TRUNCATE location_property_osmline'); + $oDB->exec('TRUNCATE location_property_osmline'); echo '.'; - $this->oDB->exec('TRUNCATE place_addressline'); + $oDB->exec('TRUNCATE place_addressline'); echo '.'; - $this->oDB->exec('TRUNCATE location_area'); + $oDB->exec('TRUNCATE location_area'); echo '.'; if (!$this->dbReverseOnly()) { - $this->oDB->exec('TRUNCATE search_name'); + $oDB->exec('TRUNCATE search_name'); echo '.'; } - $this->oDB->exec('TRUNCATE search_name_blank'); + $oDB->exec('TRUNCATE search_name_blank'); echo '.'; - $this->oDB->exec('DROP SEQUENCE seq_place'); + $oDB->exec('DROP SEQUENCE seq_place'); echo '.'; - $this->oDB->exec('CREATE SEQUENCE seq_place start 100000'); + $oDB->exec('CREATE SEQUENCE seq_place start 100000'); echo '.'; $sSQL = 'select distinct partition from country_name'; - $aPartitions = $this->oDB->getCol($sSQL); + $aPartitions = $oDB->getCol($sSQL); if (!$this->bNoPartitions) $aPartitions[] = 0; foreach ($aPartitions as $sPartition) { - $this->oDB->exec('TRUNCATE location_road_'.$sPartition); + $oDB->exec('TRUNCATE location_road_'.$sPartition); echo '.'; } // used by getorcreate_word_id to ignore frequent partial words $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - $this->oDB->exec($sSQL); + $oDB->exec($sSQL); echo ".\n"; // pre-create the word list @@ -400,13 +413,13 @@ class SetupFunctions info('Reanalysing database'); $this->pgsqlRunScript('ANALYSE'); - $sDatabaseDate = getDatabaseDate($this->oDB); - $this->oDB->exec('TRUNCATE import_status'); + $sDatabaseDate = getDatabaseDate($oDB); + $oDB->exec('TRUNCATE import_status'); if (!$sDatabaseDate) { warn('could not determine database date.'); } else { $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - $this->oDB->exec($sSQL); + $oDB->exec($sSQL); echo "Latest data imported from $sDatabaseDate.\n"; } } @@ -484,7 +497,7 @@ class SetupFunctions public function calculatePostcodes($bCMDResultAll) { info('Calculate Postcodes'); - $this->oDB->exec('TRUNCATE location_postcode'); + $this->db()->exec('TRUNCATE location_postcode'); $sSQL = 'INSERT INTO location_postcode'; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; @@ -495,7 +508,7 @@ class SetupFunctions $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; $sSQL .= ' AND geometry IS NOT null'; $sSQL .= ' GROUP BY country_code, pc'; - $this->oDB->exec($sSQL); + $this->db()->exec($sSQL); // only add postcodes that are not yet available in OSM $sSQL = 'INSERT INTO location_postcode'; @@ -505,7 +518,7 @@ class SetupFunctions $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; $sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= " WHERE country_code = 'us')"; - $this->oDB->exec($sSQL); + $this->db()->exec($sSQL); // add missing postcodes for GB (if available) $sSQL = 'INSERT INTO location_postcode'; @@ -514,61 +527,80 @@ class SetupFunctions $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; $sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= " WHERE country_code = 'gb')"; - $this->oDB->exec($sSQL); + $this->db()->exec($sSQL); if (!$bCMDResultAll) { $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; - $this->oDB->exec($sSQL); + $this->db()->exec($sSQL); } $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; - $this->oDB->exec($sSQL); + $this->db()->exec($sSQL); } public function index($bIndexNoanalyse) { - $sBaseCmd = CONST_BasePath.'/nominatim/nominatim.py' - .' -d '.escapeshellarg($this->aDSNInfo['database']) - .' -P '.escapeshellarg($this->aDSNInfo['port']) - .' -t '.escapeshellarg($this->iInstances); + checkModulePresence(); // raises exception on failure + + $oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py')) + ->addParams('--database', $this->aDSNInfo['database']) + ->addParams('--port', $this->aDSNInfo['port']) + ->addParams('--threads', $this->iInstances); + if (!$this->bQuiet) { - $sBaseCmd .= ' -v'; + $oBaseCmd->addParams('-v'); } if ($this->bVerbose) { - $sBaseCmd .= ' -v'; + $oBaseCmd->addParams('-v'); } if (isset($this->aDSNInfo['hostspec'])) { - $sBaseCmd .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']); + $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']); } if (isset($this->aDSNInfo['username'])) { - $sBaseCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oBaseCmd->addParams('--user', $this->aDSNInfo['username']); + } + if (isset($this->aDSNInfo['password'])) { + $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } info('Index ranks 0 - 4'); - $iStatus = $this->runWithPgEnv($sBaseCmd.' -R 4'); + $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4); + echo $oCmd->escapedCmd(); + + $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); + info('Index administrative boundaries'); + $oCmd = (clone $oBaseCmd)->addParams('-b'); + $iStatus = $oCmd->run(); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + info('Index ranks 5 - 25'); - $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 5 -R 25'); + $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25); + $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } + if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index ranks 26 - 30'); - $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 26'); + $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26); + $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } info('Index postcodes'); $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - $this->oDB->exec($sSQL); + $this->db()->exec($sSQL); } public function createSearchIndices() @@ -577,11 +609,11 @@ class SetupFunctions $sSQL = 'SELECT relname FROM pg_class, pg_index '; $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid'; - $aInvalidIndices = $this->oDB->getCol($sSQL); + $aInvalidIndices = $this->db()->getCol($sSQL); foreach ($aInvalidIndices as $sIndexName) { info("Cleaning up invalid index $sIndexName"); - $this->oDB->exec("DROP INDEX $sIndexName;"); + $this->db()->exec("DROP INDEX $sIndexName;"); } $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); @@ -651,7 +683,7 @@ class SetupFunctions ); $aDropTables = array(); - $aHaveTables = $this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"); + $aHaveTables = $this->db()->getListOfTables(); foreach ($aHaveTables as $sTable) { $bFound = false; @@ -670,6 +702,57 @@ class SetupFunctions $this->removeFlatnodeFile(); } + /** + * Setup settings-frontend.php in the build/website directory + * + * @return null + */ + public function setupWebsite() + { + $rOutputFile = fopen(CONST_InstallPath.'/settings/settings-frontend.php', 'w'); + + fwrite($rOutputFile, "oDB)) { + $this->oDB = new \Nominatim\DB(); + $this->oDB->connect(); + } + + return $this->oDB; + } + private function removeFlatnodeFile() { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { @@ -695,16 +778,17 @@ class SetupFunctions $sBasePath = CONST_BasePath.'/sql/functions/'; $sTemplate = file_get_contents($sBasePath.'utils.sql'); $sTemplate .= file_get_contents($sBasePath.'normalization.sql'); + $sTemplate .= file_get_contents($sBasePath.'ranking.sql'); $sTemplate .= file_get_contents($sBasePath.'importance.sql'); $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql'); $sTemplate .= file_get_contents($sBasePath.'interpolation.sql'); - if ($this->oDB->tableExists('place')) { + if ($this->db()->tableExists('place')) { $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql'); } - if ($this->oDB->tableExists('placex')) { + if ($this->db()->tableExists('placex')) { $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql'); } - if ($this->oDB->tableExists('location_postcode')) { + if ($this->db()->tableExists('location_postcode')) { $sTemplate .= file_get_contents($sBasePath.'postcode_triggers.sql'); } $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); @@ -733,7 +817,7 @@ class SetupFunctions private function pgsqlRunPartitionScript($sTemplate) { $sSQL = 'select distinct partition from country_name'; - $aPartitions = $this->oDB->getCol($sSQL); + $aPartitions = $this->db()->getCol($sSQL); if (!$this->bNoPartitions) $aPartitions[] = 0; preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); @@ -752,21 +836,21 @@ class SetupFunctions { if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - $sCMD = 'psql' - .' -p '.escapeshellarg($this->aDSNInfo['port']) - .' -d '.escapeshellarg($this->aDSNInfo['database']); + $oCmd = (new \Nominatim\Shell('psql')) + ->addParams('--port', $this->aDSNInfo['port']) + ->addParams('--dbname', $this->aDSNInfo['database']); + if (!$this->bVerbose) { - $sCMD .= ' -q'; + $oCmd->addParams('--quiet'); } if (isset($this->aDSNInfo['hostspec'])) { - $sCMD .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']); + $oCmd->addParams('--host', $this->aDSNInfo['hostspec']); } if (isset($this->aDSNInfo['username'])) { - $sCMD .= ' -U '.escapeshellarg($this->aDSNInfo['username']); + $oCmd->addParams('--username', $this->aDSNInfo['username']); } - $aProcEnv = null; if (isset($this->aDSNInfo['password'])) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); } $ahGzipPipes = null; if (preg_match('/\\.gz$/', $sFilename)) { @@ -775,12 +859,14 @@ class SetupFunctions 1 => array('pipe', 'w'), 2 => array('file', '/dev/null', 'a') ); - $hGzipProcess = proc_open('zcat '.escapeshellarg($sFilename), $aDescriptors, $ahGzipPipes); + $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename); + + $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes); if (!is_resource($hGzipProcess)) fail('unable to start zcat'); $aReadPipe = $ahGzipPipes[1]; fclose($ahGzipPipes[0]); } else { - $sCMD .= ' -f '.escapeshellarg($sFilename); + $oCmd->addParams('--file', $sFilename); $aReadPipe = array('pipe', 'r'); } $aDescriptors = array( @@ -789,7 +875,8 @@ class SetupFunctions 2 => array('file', '/dev/null', 'a') ); $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); + + $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv); if (!is_resource($hProcess)) fail('unable to start pgsql'); // TODO: error checking while (!feof($ahPipes[1])) { @@ -830,34 +917,17 @@ class SetupFunctions return $sSql; } - private function runWithPgEnv($sCmd) - { - if ($this->bVerbose) { - echo "Execute: $sCmd\n"; - } - - $aProcEnv = null; - - if (isset($this->aDSNInfo['password'])) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - - return runWithEnv($sCmd, $aProcEnv); - } - /** * Drop table with the given name if it exists. * * @param string $sName Name of table to remove. * * @return null - * - * @pre connect() must have been called. */ private function dropTable($sName) { if ($this->bVerbose) echo "Dropping table $sName\n"; - $this->oDB->exec('DROP TABLE IF EXISTS '.$sName.' CASCADE'); + $this->db()->deleteTable($sName); } /** @@ -867,6 +937,6 @@ class SetupFunctions */ private function dbReverseOnly() { - return !($this->oDB->tableExists('search_name')); + return !($this->db()->tableExists('search_name')); } }