X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/f6e894a53af83a69f553555cb4a6248d57a58391..0ee821d64f7fe700e593583f74141e1edafaacc6:/lib-php/setup/SetupClass.php diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php index 1e7dccb4..d07adce7 100755 --- a/lib-php/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -6,7 +6,6 @@ require_once(CONST_LibDir.'/Shell.php'); class SetupFunctions { - protected $iCacheMemory; protected $iInstances; protected $aDSNInfo; protected $bQuiet; @@ -14,7 +13,6 @@ class SetupFunctions protected $sIgnoreErrors; protected $bEnableDiffUpdates; protected $bEnableDebugStatements; - protected $bNoPartitions; protected $bDrop; protected $oDB = null; protected $oNominatimCmd; @@ -31,16 +29,6 @@ class SetupFunctions warn('resetting threads to '.$this->iInstances); } - if (isset($aCMDResult['osm2pgsql-cache'])) { - $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; - } elseif (getSetting('FLATNODE_FILE')) { - // When flatnode files are enabled then disable cache per default. - $this->iCacheMemory = 0; - } else { - // Otherwise: Assume we can steal all the cache memory in the box. - $this->iCacheMemory = getCacheMemoryMB(); - } - // parse database string $this->aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN')); if (!isset($this->aDSNInfo['port'])) { @@ -62,11 +50,6 @@ class SetupFunctions } else { $this->bEnableDebugStatements = false; } - if (isset($aCMDResult['no-partitions'])) { - $this->bNoPartitions = $aCMDResult['no-partitions']; - } else { - $this->bNoPartitions = false; - } if (isset($aCMDResult['enable-diff-updates'])) { $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; } else { @@ -84,329 +67,6 @@ class SetupFunctions } } - public function importData($sOSMFile) - { - info('Import data'); - - if (!file_exists(getOsm2pgsqlBinary())) { - echo "Check NOMINATIM_OSM2PGSQL_BINARY in your local .env file.\n"; - echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '".getOsm2pgsqlBinary()."'"); - } - - $oCmd = new \Nominatim\Shell(getOsm2pgsqlBinary()); - $oCmd->addParams('--style', getImportStyle()); - - if (getSetting('FLATNODE_FILE')) { - $oCmd->addParams('--flat-nodes', getSetting('FLATNODE_FILE')); - } - if (getSetting('TABLESPACE_OSM_DATA')) { - $oCmd->addParams('--tablespace-slim-data', getSetting('TABLESPACE_OSM_DATA')); - } - if (getSetting('TABLESPACE_OSM_INDEX')) { - $oCmd->addParams('--tablespace-slim-index', getSetting('TABLESPACE_OSM_INDEX')); - } - if (getSetting('TABLESPACE_PLACE_DATA')) { - $oCmd->addParams('--tablespace-main-data', getSetting('TABLESPACE_PLACE_DATA')); - } - if (getSetting('TABLESPACE_PLACE_INDEX')) { - $oCmd->addParams('--tablespace-main-index', getSetting('TABLESPACE_PLACE_INDEX')); - } - $oCmd->addParams('--latlong', '--slim', '--create'); - $oCmd->addParams('--output', 'gazetteer'); - $oCmd->addParams('--hstore'); - $oCmd->addParams('--number-processes', 1); - $oCmd->addParams('--with-forward-dependencies', 'false'); - $oCmd->addParams('--log-progress', 'true'); - $oCmd->addParams('--cache', $this->iCacheMemory); - $oCmd->addParams('--port', $this->aDSNInfo['port']); - - if (isset($this->aDSNInfo['username'])) { - $oCmd->addParams('--username', $this->aDSNInfo['username']); - } - if (isset($this->aDSNInfo['password'])) { - $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); - } - if (isset($this->aDSNInfo['hostspec'])) { - $oCmd->addParams('--host', $this->aDSNInfo['hostspec']); - } - $oCmd->addParams('--database', $this->aDSNInfo['database']); - $oCmd->addParams($sOSMFile); - $oCmd->run(); - - if (!$this->sIgnoreErrors && !$this->db()->getRow('select * from place limit 1')) { - fail('No Data'); - } - - if ($this->bDrop) { - $this->dropTable('planet_osm_nodes'); - $this->removeFlatnodeFile(); - } - } - - public function createFunctions() - { - info('Create Functions'); - - // Try accessing the C module, so we know early if something is wrong - $this->checkModulePresence(); // raises exception on failure - - $this->createSqlFunctions(); - } - - public function createTables($bReverseOnly = false) - { - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate, false); - - if ($bReverseOnly) { - $this->dropTable('search_name'); - } - - (clone($this->oNominatimCmd))->addParams('refresh', '--address-levels')->run(); - } - - public function createTableTriggers() - { - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate, false); - } - - public function createPartitionTables() - { - info('Create Partition Tables'); - - $sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunPartitionScript($sTemplate); - } - - public function createPartitionFunctions() - { - info('Create Partition Functions'); - $this->createSqlFunctions(); // also create partition functions - } - - public function importWikipediaArticles() - { - $sWikiArticlePath = getSetting('WIKIPEDIA_DATA_PATH', CONST_InstallDir); - $sWikiArticlesFile = $sWikiArticlePath.'/wikimedia-importance.sql.gz'; - if (file_exists($sWikiArticlesFile)) { - info('Importing wikipedia articles and redirects'); - $this->dropTable('wikipedia_article'); - $this->dropTable('wikipedia_redirect'); - $this->pgsqlRunScriptFile($sWikiArticlesFile); - } else { - warn('wikipedia importance dump file not found - places will have default importance'); - } - } - - public function loadData($bDisableTokenPrecalc) - { - info('Drop old Data'); - - $oDB = $this->db(); - - $oDB->exec('TRUNCATE word'); - echo '.'; - $oDB->exec('TRUNCATE placex'); - echo '.'; - $oDB->exec('TRUNCATE location_property_osmline'); - echo '.'; - $oDB->exec('TRUNCATE place_addressline'); - echo '.'; - $oDB->exec('TRUNCATE location_area'); - echo '.'; - if (!$this->dbReverseOnly()) { - $oDB->exec('TRUNCATE search_name'); - echo '.'; - } - $oDB->exec('TRUNCATE search_name_blank'); - echo '.'; - $oDB->exec('DROP SEQUENCE seq_place'); - echo '.'; - $oDB->exec('CREATE SEQUENCE seq_place start 100000'); - echo '.'; - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = $oDB->getCol($sSQL); - - if (!$this->bNoPartitions) $aPartitions[] = 0; - foreach ($aPartitions as $sPartition) { - $oDB->exec('TRUNCATE location_road_'.$sPartition); - echo '.'; - } - - // used by getorcreate_word_id to ignore frequent partial words - $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT '.getSetting('MAX_WORD_FREQUENCY').' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - $oDB->exec($sSQL); - echo ".\n"; - - // pre-create the word list - if (!$bDisableTokenPrecalc) { - info('Loading word list'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/words.sql'); - } - - info('Load Data'); - $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; - - $aDBInstances = array(); - $iLoadThreads = max(1, $this->iInstances - 1); - for ($i = 0; $i < $iLoadThreads; $i++) { - // https://secure.php.net/manual/en/function.pg-connect.php - $DSN = getSetting('DATABASE_DSN'); - $DSN = preg_replace('/^pgsql:/', '', $DSN); - $DSN = preg_replace('/;/', ' ', $DSN); - $aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW); - pg_ping($aDBInstances[$i]); - } - - for ($i = 0; $i < $iLoadThreads; $i++) { - $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; - $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; - $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; - $sSQL .= ' and ST_IsValid(geometry)'; - if ($this->bVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$i], $sSQL)) { - fail(pg_last_error($aDBInstances[$i])); - } - } - - // last thread for interpolation lines - // https://secure.php.net/manual/en/function.pg-connect.php - $DSN = getSetting('DATABASE_DSN'); - $DSN = preg_replace('/^pgsql:/', '', $DSN); - $DSN = preg_replace('/;/', ' ', $DSN); - $aDBInstances[$iLoadThreads] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW); - pg_ping($aDBInstances[$iLoadThreads]); - $sSQL = 'insert into location_property_osmline'; - $sSQL .= ' (osm_id, address, linegeo)'; - $sSQL .= ' SELECT osm_id, address, geometry from place where '; - $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($this->bVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$iLoadThreads], $sSQL)) { - fail(pg_last_error($aDBInstances[$iLoadThreads])); - } - - $bFailed = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - while (($hPGresult = pg_get_result($aDBInstances[$i])) !== false) { - $resultStatus = pg_result_status($hPGresult); - // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, - // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, - // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - // echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; - if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { - $resultError = pg_result_error($hPGresult); - echo '-- error text ' . $i . ': ' . $resultError . "\n"; - $bFailed = true; - } - } - } - if ($bFailed) { - fail('SQL errors loading placex and/or location_property_osmline tables'); - } - - for ($i = 0; $i < $this->iInstances; $i++) { - pg_close($aDBInstances[$i]); - } - - echo "\n"; - info('Reanalysing database'); - $this->pgsqlRunScript('ANALYSE'); - - $sDatabaseDate = getDatabaseDate($oDB); - $oDB->exec('TRUNCATE import_status'); - if (!$sDatabaseDate) { - warn('could not determine database date.'); - } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - $oDB->exec($sSQL); - echo "Latest data imported from $sDatabaseDate.\n"; - } - } - - public function importTigerData($sTigerPath) - { - info('Import Tiger data'); - - $aFilenames = glob($sTigerPath.'/*.sql'); - info('Found '.count($aFilenames).' SQL files in path '.$sTigerPath); - if (empty($aFilenames)) { - warn('Tiger data import selected but no files found in path '.$sTigerPath); - return; - } - $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_start.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate, false); - - $aDBInstances = array(); - for ($i = 0; $i < $this->iInstances; $i++) { - // https://secure.php.net/manual/en/function.pg-connect.php - $DSN = getSetting('DATABASE_DSN'); - $DSN = preg_replace('/^pgsql:/', '', $DSN); - $DSN = preg_replace('/;/', ' ', $DSN); - $aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW | PGSQL_CONNECT_ASYNC); - pg_ping($aDBInstances[$i]); - } - - foreach ($aFilenames as $sFile) { - echo $sFile.': '; - $hFile = fopen($sFile, 'r'); - $sSQL = fgets($hFile, 100000); - $iLines = 0; - while (true) { - for ($i = 0; $i < $this->iInstances; $i++) { - if (!pg_connection_busy($aDBInstances[$i])) { - while (pg_get_result($aDBInstances[$i])); - $sSQL = fgets($hFile, 100000); - if (!$sSQL) break 2; - if (!pg_send_query($aDBInstances[$i], $sSQL)) fail(pg_last_error($aDBInstances[$i])); - $iLines++; - if ($iLines == 1000) { - echo '.'; - $iLines = 0; - } - } - } - usleep(10); - } - fclose($hFile); - - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i < $this->iInstances; $i++) { - if (pg_connection_busy($aDBInstances[$i])) $bAnyBusy = true; - } - usleep(10); - } - echo "\n"; - } - - for ($i = 0; $i < $this->iInstances; $i++) { - pg_close($aDBInstances[$i]); - } - - info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_finish.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate, false); - } - public function calculatePostcodes($bCMDResultAll) { info('Calculate Postcodes'); @@ -470,116 +130,6 @@ class SetupFunctions $this->db()->exec($sSQL); } - public function index($bIndexNoanalyse) - { - $this->checkModulePresence(); // raises exception on failure - - $oBaseCmd = (clone $this->oNominatimCmd)->addParams('index'); - - info('Index ranks 0 - 4'); - $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4); - - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - - info('Index administrative boundaries'); - $oCmd = (clone $oBaseCmd)->addParams('--boundaries-only'); - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - info('Index ranks 5 - 25'); - $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 5, '--maxrank', 25); - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - - info('Index ranks 26 - 30'); - $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 26); - $iStatus = $oCmd->run(); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - info('Index postcodes'); - $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - $this->db()->exec($sSQL); - } - - public function createSearchIndices() - { - info('Create Search indices'); - - $sSQL = 'SELECT relname FROM pg_class, pg_index '; - $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid'; - $aInvalidIndices = $this->db()->getCol($sSQL); - - foreach ($aInvalidIndices as $sIndexName) { - info("Cleaning up invalid index $sIndexName"); - $this->db()->exec("DROP INDEX $sIndexName;"); - } - - $sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql'); - if (!$this->bDrop) { - $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql'); - } - if (!$this->dbReverseOnly()) { - $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql'); - } - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate); - } - - public function createCountryNames() - { - info('Create search index for default country names'); - - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); - $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); - $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' - .'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; - $sLanguages = getSetting('LANGUAGES'); - if ($sLanguages) { - $sSQL .= 'in '; - $sDelim = '('; - foreach (explode(',', $sLanguages) as $sLang) { - $sSQL .= $sDelim."'name:$sLang'"; - $sDelim = ','; - } - $sSQL .= ')'; - } else { - // all include all simple name tags - $sSQL .= "like 'name:%'"; - } - $sSQL .= ') v'; - $this->pgsqlRunScript($sSQL); - } - - public function drop() - { - (clone($this->oNominatimCmd))->addParams('freeze')->run(); - } - - /** - * Setup the directory for the API scripts. - * - * @return null - */ - public function setupWebsite() - { - (clone($this->oNominatimCmd))->addParams('refresh', '--website')->run(); - } - /** * Return the connection to the database. * @@ -598,15 +148,6 @@ class SetupFunctions return $this->oDB; } - private function removeFlatnodeFile() - { - $sFName = getSetting('FLATNODE_FILE'); - if ($sFName && file_exists($sFName)) { - if ($this->bVerbose) echo 'Deleting '.$sFName."\n"; - unlink($sFName); - } - } - private function pgsqlRunScript($sScript, $bfatal = true) { runSQLScript( @@ -617,7 +158,7 @@ class SetupFunctions ); } - private function createSqlFunctions() + public function createSqlFunctions() { $oCmd = (clone($this->oNominatimCmd)) ->addParams('refresh', '--functions'); @@ -630,25 +171,7 @@ class SetupFunctions $oCmd->addParams('--enable-debug-statements'); } - $oCmd->run(); - } - - private function pgsqlRunPartitionScript($sTemplate) - { - $sSQL = 'select distinct partition from country_name'; - $aPartitions = $this->db()->getCol($sSQL); - if (!$this->bNoPartitions) $aPartitions[] = 0; - - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach ($aMatches as $aMatch) { - $sResult = ''; - foreach ($aPartitions as $sPartitionName) { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - - $this->pgsqlRunScript($sTemplate); + $oCmd->run(!$this->sIgnoreErrors); } private function pgsqlRunScriptFile($sFilename) @@ -735,44 +258,4 @@ class SetupFunctions return $sSql; } - - /** - * Drop table with the given name if it exists. - * - * @param string $sName Name of table to remove. - * - * @return null - */ - private function dropTable($sName) - { - if ($this->bVerbose) echo "Dropping table $sName\n"; - $this->db()->deleteTable($sName); - } - - /** - * Check if the database is in reverse-only mode. - * - * @return True if there is no search_name table and infrastructure. - */ - private function dbReverseOnly() - { - return !($this->db()->tableExists('search_name')); - } - - /** - * Try accessing the C module, so we know early if something is wrong. - * - * Raises Nominatim\DatabaseError on failure - */ - private function checkModulePresence() - { - $sModulePath = getSetting('DATABASE_MODULE_PATH', CONST_InstallDir.'/module'); - $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; - $sSQL .= $sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; - $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; - - $oDB = new \Nominatim\DB(); - $oDB->connect(); - $oDB->exec($sSQL, null, 'Database server failed to load '.$sModulePath.'/nominatim.so module'); - } }