X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/db3ced17bbfff00411f506d8c84419c875959d5e..57db5819efb1eac7497aca2dbdb2493a67bd33f9:/lib-php/setup/SetupClass.php?ds=inline diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php index 2a498287..b0081fd8 100755 --- a/lib-php/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -6,7 +6,6 @@ require_once(CONST_LibDir.'/Shell.php'); class SetupFunctions { - protected $iCacheMemory; protected $iInstances; protected $aDSNInfo; protected $bQuiet; @@ -31,16 +30,6 @@ class SetupFunctions warn('resetting threads to '.$this->iInstances); } - if (isset($aCMDResult['osm2pgsql-cache'])) { - $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; - } elseif (getSetting('FLATNODE_FILE')) { - // When flatnode files are enabled then disable cache per default. - $this->iCacheMemory = 0; - } else { - // Otherwise: Assume we can steal all the cache memory in the box. - $this->iCacheMemory = getCacheMemoryMB(); - } - // parse database string $this->aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN')); if (!isset($this->aDSNInfo['port'])) { @@ -84,161 +73,6 @@ class SetupFunctions } } - public function createDB() - { - info('Create DB'); - $oDB = new \Nominatim\DB; - - if ($oDB->checkConnection()) { - fail('database already exists ('.getSetting('DATABASE_DSN').')'); - } - - $oCmd = (new \Nominatim\Shell('createdb')) - ->addParams('-E', 'UTF-8') - ->addParams('-p', $this->aDSNInfo['port']); - - if (isset($this->aDSNInfo['username'])) { - $oCmd->addParams('-U', $this->aDSNInfo['username']); - } - if (isset($this->aDSNInfo['password'])) { - $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); - } - if (isset($this->aDSNInfo['hostspec'])) { - $oCmd->addParams('-h', $this->aDSNInfo['hostspec']); - } - $oCmd->addParams($this->aDSNInfo['database']); - - $result = $oCmd->run(); - if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd()); - } - - public function setupDB() - { - info('Setup DB'); - - $fPostgresVersion = $this->db()->getPostgresVersion(); - echo 'Postgres version found: '.$fPostgresVersion."\n"; - - if ($fPostgresVersion < 9.03) { - fail('Minimum supported version of Postgresql is 9.3.'); - } - - $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); - $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); - - $fPostgisVersion = $this->db()->getPostgisVersion(); - echo 'Postgis version found: '.$fPostgisVersion."\n"; - - if ($fPostgisVersion < 2.2) { - echo "Minimum required Postgis version 2.2\n"; - exit(1); - } - - $sPgUser = getSetting('DATABASE_WEBUSER'); - $i = $this->db()->getOne("select count(*) from pg_user where usename = '$sPgUser'"); - if ($i == 0) { - echo "\nERROR: Web user '".$sPgUser."' does not exist. Create it with:\n"; - echo "\n createuser ".$sPgUser."\n\n"; - exit(1); - } - - if (!getSetting('DATABASE_MODULE_PATH')) { - // If no custom module path is set then copy the module into the - // project directory, but only if it is not the same file already - // (aka we are running from the build dir). - $sDest = CONST_InstallDir.'/module'; - if ($sDest != CONST_Default_ModulePath) { - if (!file_exists($sDest)) { - mkdir($sDest); - } - if (!copy(CONST_Default_ModulePath.'/nominatim.so', $sDest.'/nominatim.so')) { - echo "Failed to copy database module to $sDest."; - exit(1); - } - chmod($sDest.'/nominatim.so', 0755); - info("Database module installed at $sDest."); - } else { - info('Running from build directory. Leaving database module as is.'); - } - } else { - info('Using database module from DATABASE_MODULE_PATH ('.getSetting('DATABASE_MODULE_PATH').').'); - } - // Try accessing the C module, so we know early if something is wrong - $this->checkModulePresence(); // raises exception on failure - - if (!file_exists(CONST_DataDir.'/data/country_osm_grid.sql.gz')) { - echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O ".CONST_DataDir."/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; - exit(1); - } - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/country_name.sql'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/country_osm_grid.sql.gz'); - - if ($this->bNoPartitions) { - $this->pgsqlRunScript('update country_name set partition = 0'); - } - } - - public function importData($sOSMFile) - { - info('Import data'); - - if (!file_exists(getOsm2pgsqlBinary())) { - echo "Check NOMINATIM_OSM2PGSQL_BINARY in your local .env file.\n"; - echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '".getOsm2pgsqlBinary()."'"); - } - - $oCmd = new \Nominatim\Shell(getOsm2pgsqlBinary()); - $oCmd->addParams('--style', getImportStyle()); - - if (getSetting('FLATNODE_FILE')) { - $oCmd->addParams('--flat-nodes', getSetting('FLATNODE_FILE')); - } - if (getSetting('TABLESPACE_OSM_DATA')) { - $oCmd->addParams('--tablespace-slim-data', getSetting('TABLESPACE_OSM_DATA')); - } - if (getSetting('TABLESPACE_OSM_INDEX')) { - $oCmd->addParams('--tablespace-slim-index', getSetting('TABLESPACE_OSM_INDEX')); - } - if (getSetting('TABLESPACE_PLACE_DATA')) { - $oCmd->addParams('--tablespace-main-data', getSetting('TABLESPACE_PLACE_DATA')); - } - if (getSetting('TABLESPACE_PLACE_INDEX')) { - $oCmd->addParams('--tablespace-main-index', getSetting('TABLESPACE_PLACE_INDEX')); - } - $oCmd->addParams('--latlong', '--slim', '--create'); - $oCmd->addParams('--output', 'gazetteer'); - $oCmd->addParams('--hstore'); - $oCmd->addParams('--number-processes', 1); - $oCmd->addParams('--with-forward-dependencies', 'false'); - $oCmd->addParams('--log-progress', 'true'); - $oCmd->addParams('--cache', $this->iCacheMemory); - $oCmd->addParams('--port', $this->aDSNInfo['port']); - - if (isset($this->aDSNInfo['username'])) { - $oCmd->addParams('--username', $this->aDSNInfo['username']); - } - if (isset($this->aDSNInfo['password'])) { - $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); - } - if (isset($this->aDSNInfo['hostspec'])) { - $oCmd->addParams('--host', $this->aDSNInfo['hostspec']); - } - $oCmd->addParams('--database', $this->aDSNInfo['database']); - $oCmd->addParams($sOSMFile); - $oCmd->run(); - - if (!$this->sIgnoreErrors && !$this->db()->getRow('select * from place limit 1')) { - fail('No Data'); - } - - if ($this->bDrop) { - $this->dropTable('planet_osm_nodes'); - $this->removeFlatnodeFile(); - } - } - public function createFunctions() { info('Create Functions'); @@ -253,7 +87,7 @@ class SetupFunctions { info('Create Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tables.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -269,7 +103,7 @@ class SetupFunctions { info('Create Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/table-triggers.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -279,159 +113,12 @@ class SetupFunctions { info('Create Partition Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/partition-tables.src.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunPartitionScript($sTemplate); } - public function createPartitionFunctions() - { - info('Create Partition Functions'); - $this->createSqlFunctions(); // also create partition functions - } - - public function importWikipediaArticles() - { - $sWikiArticlePath = getSetting('WIKIPEDIA_DATA_PATH', CONST_InstallDir); - $sWikiArticlesFile = $sWikiArticlePath.'/wikimedia-importance.sql.gz'; - if (file_exists($sWikiArticlesFile)) { - info('Importing wikipedia articles and redirects'); - $this->dropTable('wikipedia_article'); - $this->dropTable('wikipedia_redirect'); - $this->pgsqlRunScriptFile($sWikiArticlesFile); - } else { - warn('wikipedia importance dump file not found - places will have default importance'); - } - } - - public function loadData($bDisableTokenPrecalc) - { - info('Drop old Data'); - - $oDB = $this->db(); - - $oDB->exec('TRUNCATE word'); - echo '.'; - $oDB->exec('TRUNCATE placex'); - echo '.'; - $oDB->exec('TRUNCATE location_property_osmline'); - echo '.'; - $oDB->exec('TRUNCATE place_addressline'); - echo '.'; - $oDB->exec('TRUNCATE location_area'); - echo '.'; - if (!$this->dbReverseOnly()) { - $oDB->exec('TRUNCATE search_name'); - echo '.'; - } - $oDB->exec('TRUNCATE search_name_blank'); - echo '.'; - $oDB->exec('DROP SEQUENCE seq_place'); - echo '.'; - $oDB->exec('CREATE SEQUENCE seq_place start 100000'); - echo '.'; - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = $oDB->getCol($sSQL); - - if (!$this->bNoPartitions) $aPartitions[] = 0; - foreach ($aPartitions as $sPartition) { - $oDB->exec('TRUNCATE location_road_'.$sPartition); - echo '.'; - } - - // used by getorcreate_word_id to ignore frequent partial words - $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT '.getSetting('MAX_WORD_FREQUENCY').' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - $oDB->exec($sSQL); - echo ".\n"; - - // pre-create the word list - if (!$bDisableTokenPrecalc) { - info('Loading word list'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/words.sql'); - } - - info('Load Data'); - $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; - - $aDBInstances = array(); - $iLoadThreads = max(1, $this->iInstances - 1); - for ($i = 0; $i < $iLoadThreads; $i++) { - // https://secure.php.net/manual/en/function.pg-connect.php - $DSN = getSetting('DATABASE_DSN'); - $DSN = preg_replace('/^pgsql:/', '', $DSN); - $DSN = preg_replace('/;/', ' ', $DSN); - $aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW); - pg_ping($aDBInstances[$i]); - } - - for ($i = 0; $i < $iLoadThreads; $i++) { - $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; - $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; - $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; - $sSQL .= ' and ST_IsValid(geometry)'; - if ($this->bVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$i], $sSQL)) { - fail(pg_last_error($aDBInstances[$i])); - } - } - - // last thread for interpolation lines - // https://secure.php.net/manual/en/function.pg-connect.php - $DSN = getSetting('DATABASE_DSN'); - $DSN = preg_replace('/^pgsql:/', '', $DSN); - $DSN = preg_replace('/;/', ' ', $DSN); - $aDBInstances[$iLoadThreads] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW); - pg_ping($aDBInstances[$iLoadThreads]); - $sSQL = 'insert into location_property_osmline'; - $sSQL .= ' (osm_id, address, linegeo)'; - $sSQL .= ' SELECT osm_id, address, geometry from place where '; - $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($this->bVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$iLoadThreads], $sSQL)) { - fail(pg_last_error($aDBInstances[$iLoadThreads])); - } - - $bFailed = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - while (($hPGresult = pg_get_result($aDBInstances[$i])) !== false) { - $resultStatus = pg_result_status($hPGresult); - // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, - // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, - // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - // echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; - if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { - $resultError = pg_result_error($hPGresult); - echo '-- error text ' . $i . ': ' . $resultError . "\n"; - $bFailed = true; - } - } - } - if ($bFailed) { - fail('SQL errors loading placex and/or location_property_osmline tables'); - } - - for ($i = 0; $i < $this->iInstances; $i++) { - pg_close($aDBInstances[$i]); - } - - echo "\n"; - info('Reanalysing database'); - $this->pgsqlRunScript('ANALYSE'); - - $sDatabaseDate = getDatabaseDate($oDB); - $oDB->exec('TRUNCATE import_status'); - if (!$sDatabaseDate) { - warn('could not determine database date.'); - } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - $oDB->exec($sSQL); - echo "Latest data imported from $sDatabaseDate.\n"; - } - } - public function importTigerData($sTigerPath) { info('Import Tiger data'); @@ -442,7 +129,7 @@ class SetupFunctions warn('Tiger data import selected but no files found in path '.$sTigerPath); return; } - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_start.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_start.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -496,7 +183,7 @@ class SetupFunctions } info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_finish.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_finish.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -505,7 +192,7 @@ class SetupFunctions public function calculatePostcodes($bCMDResultAll) { info('Calculate Postcodes'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/sql/postcode_tables.sql'); + $this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql'); $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz'; if (file_exists($sPostcodeFilename)) { @@ -565,49 +252,6 @@ class SetupFunctions $this->db()->exec($sSQL); } - public function index($bIndexNoanalyse) - { - $this->checkModulePresence(); // raises exception on failure - - $oBaseCmd = (clone $this->oNominatimCmd)->addParams('index'); - - info('Index ranks 0 - 4'); - $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4); - - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - - info('Index administrative boundaries'); - $oCmd = (clone $oBaseCmd)->addParams('--boundaries-only'); - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - info('Index ranks 5 - 25'); - $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 5, '--maxrank', 25); - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - - info('Index ranks 26 - 30'); - $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 26); - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - info('Index postcodes'); - $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - $this->db()->exec($sSQL); - } - public function createSearchIndices() { info('Create Search indices'); @@ -621,12 +265,12 @@ class SetupFunctions $this->db()->exec("DROP INDEX $sIndexName;"); } - $sTemplate = file_get_contents(CONST_DataDir.'/sql/indices.src.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql'); if (!$this->bDrop) { - $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_updates.src.sql'); + $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql'); } if (!$this->dbReverseOnly()) { - $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_search.src.sql'); + $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql'); } $sTemplate = $this->replaceSqlPatterns($sTemplate); @@ -660,107 +304,6 @@ class SetupFunctions $this->pgsqlRunScript($sSQL); } - public function drop() - { - info('Drop tables only required for updates'); - - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*', - 'country_osm_grid' - ); - - $aDropTables = array(); - $aHaveTables = $this->db()->getListOfTables(); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - foreach ($aDropTables as $sDrop) { - $this->dropTable($sDrop); - } - - $this->removeFlatnodeFile(); - } - - /** - * Setup the directory for the API scripts. - * - * @return null - */ - public function setupWebsite() - { - if (!is_dir(CONST_InstallDir.'/website')) { - info('Creating directory for website scripts at: '.CONST_InstallDir.'/website'); - mkdir(CONST_InstallDir.'/website'); - } - - $aScripts = array( - 'deletable.php', - 'details.php', - 'lookup.php', - 'polygons.php', - 'reverse.php', - 'search.php', - 'status.php' - ); - - foreach ($aScripts as $sScript) { - $rFile = fopen(CONST_InstallDir.'/website/'.$sScript, 'w'); - - fwrite($rFile, "oDB; } - private function removeFlatnodeFile() - { - $sFName = getSetting('FLATNODE_FILE'); - if ($sFName && file_exists($sFName)) { - if ($this->bVerbose) echo 'Deleting '.$sFName."\n"; - unlink($sFName); - } - } - private function pgsqlRunScript($sScript, $bfatal = true) { runSQLScript( @@ -811,7 +345,7 @@ class SetupFunctions $oCmd->addParams('--enable-debug-statements'); } - $oCmd->run(); + $oCmd->run(!$this->sIgnoreErrors); } private function pgsqlRunPartitionScript($sTemplate)