X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/a3b4f80c996056136828560e8e5fb26db367f238..c5109d39d029fb013286e4ff4220e200267caa09:/utils/update.php?ds=sidebyside diff --git a/utils/update.php b/utils/update.php index 176bde06..31f32f60 100755 --- a/utils/update.php +++ b/utils/update.php @@ -1,42 +1,69 @@ #!@PHP_BIN@ -Cq getTotalMemoryMB()) { $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } -$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port']; +$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; } @@ -49,12 +76,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { } if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; + $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } if ($aResult['init-updates']) { // sanity check that the replication URL is correct - $sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt'); + $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt'); if ($sBaseState === false) { echo "\nCannot find state.txt file at the configured replication URL.\n"; echo "Does the URL point to a directory containing OSM update data?\n\n"; @@ -68,7 +95,7 @@ if ($aResult['init-updates']) { fail('CONST_Pyosmium_Binary not configured'); } $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary . ' --help'; + $sCmd = CONST_Pyosmium_Binary.' --help'; exec($sCmd, $aOutput, $iRet); if ($iRet != 0) { echo "Cannot execute pyosmium-get-changes.\n"; @@ -78,9 +105,11 @@ if ($aResult['init-updates']) { } if (!$aResult['no-update-functions']) { - // instatiate setupClass to use the function therein - // instantiate Setup class - $cSetup = new SetupFunctions($aResult); + // instantiate setupClass to use the function therein + $cSetup = new SetupFunctions(array( + 'enable-diff-updates' => true, + 'verbose' => $aResult['verbose'] + )); $cSetup->createFunctions(); } @@ -88,11 +117,11 @@ if ($aResult['init-updates']) { if ($sDatabaseDate === false) { fail('Cannot determine date of database.'); } - $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60)); + $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60)); // get the appropriate state id $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url; + $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url; exec($sCmd, $aOutput, $iRet); if ($iRet != 0 || $aOutput[0] == 'None') { fail('Error running pyosmium tools'); @@ -100,7 +129,7 @@ if ($aResult['init-updates']) { pg_query($oDB->connection, 'TRUNCATE import_status'); $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('"; - $sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)'; + $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)'; if (!pg_query($oDB->connection, $sSQL)) { fail('Could not enter sequence into database.'); } @@ -115,7 +144,7 @@ if ($aResult['check-for-updates']) { fail('Updates not set up. Please run ./utils/update.php --init-updates.'); } - system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet); + system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet); exit($iRet); } @@ -128,8 +157,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { } // Import the file - $sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile; - echo $sCMD . "\n"; + $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile; + echo $sCMD."\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { @@ -141,35 +170,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { if ($aResult['calculate-postcodes']) { info('Update postcodes centroids'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql'); runSQLScript($sTemplate, true, true); } -$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc'; +$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc'; $bHaveDiff = false; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; $sContentURL = ''; if (isset($aResult['import-node']) && $aResult['import-node']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node']; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;'; } } if (isset($aResult['import-way']) && $aResult['import-way']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full'; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;'; } } if (isset($aResult['import-relation']) && $aResult['import-relation']) { if ($bUseOSMApi) { - $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full'; + $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;'; } } @@ -180,8 +209,8 @@ if ($sContentURL) { if ($bHaveDiff) { // import generated change file - $sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile; - echo $sCMD . "\n"; + $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile; + echo $sCMD."\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { fail("osm2pgsql exited with error level $iErrorLevel\n"); @@ -189,7 +218,7 @@ if ($bHaveDiff) { } if ($aResult['deduplicate']) { - $oDB = &getDB(); + $oDB =& getDB(); if (getPostgresVersion($oDB) < 9.3) { fail('ERROR: deduplicate is only currently supported in postgresql 9.3'); @@ -201,7 +230,7 @@ if ($aResult['deduplicate']) { // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates foreach ($aPartitions as $i => $sPartition) { - $sSQL = 'select count(*) from search_name_' . $sPartition; + $sSQL = 'select count(*) from search_name_'.$sPartition; $nEntries = chksql($oDB->getOne($sSQL)); if ($nEntries == 0) { unset($aPartitions[$i]); @@ -213,14 +242,11 @@ if ($aResult['deduplicate']) { $sSQL .= ' group by word_token having count(*) > 1 order by word_token'; $aDuplicateTokens = chksql($oDB->getAll($sSQL)); foreach ($aDuplicateTokens as $aToken) { - if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') { - continue; - } - - echo 'Deduping ' . $aToken['word_token'] . "\n"; + if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; + echo 'Deduping '.$aToken['word_token']."\n"; $sSQL = 'select word_id,'; $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num'; - $sSQL .= " from word where word_token = '" . $aToken['word_token']; + $sSQL .= " from word where word_token = '".$aToken['word_token']; $sSQL .= "' and class is null and type is null and country_code is null order by num desc"; $aTokenSet = chksql($oDB->getAll($sSQL)); @@ -229,34 +255,34 @@ if ($aResult['deduplicate']) { foreach ($aTokenSet as $aRemove) { $sSQL = 'update search_name set'; - $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); $sSQL = 'update search_name set'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); foreach ($aPartitions as $sPartition) { - $sSQL = 'update search_name_' . $sPartition . ' set'; - $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL = 'update search_name_'.$sPartition.' set'; + $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); } - $sSQL = 'delete from word where word_id = ' . $aRemove['word_id']; + $sSQL = 'delete from word where word_id = '.$aRemove['word_id']; chksql($oDB->query($sSQL)); } } @@ -264,12 +290,12 @@ if ($aResult['deduplicate']) { if ($aResult['recompute-word-counts']) { info('Recompute frequency of full-word search terms'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); runSQLScript($sTemplate, true, true); } if ($aResult['index']) { - $sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank']; + $sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCmd .= ' -H ' . $aDSNInfo['hostspec']; } @@ -284,13 +310,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { fail('Error: Update interval too low for download.geofabrik.de. ' . - "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); + "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); } - $sImportFile = CONST_InstallPath . '/osmosischange.osc'; - $sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size; - $sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile; - $sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances']; + $sImportFile = CONST_InstallPath.'/osmosischange.osc'; + $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size; + $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; + $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec']; } @@ -307,7 +333,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { exit(1); } - echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n"; + echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n"; $sBatchEnd = $aLastState['lastimportdate']; $iEndSequence = $aLastState['sequence_id']; @@ -330,23 +356,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (file_exists($sImportFile)) { unlink($sImportFile); } - exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult); + exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult); if ($iResult == 3) { - echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n"; + echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n"; sleep(CONST_Replication_Recheck_Interval); } elseif ($iResult != 0) { echo 'ERROR: updates failed.'; exit($iResult); } else { - $iEndSequence = (int) $aOutput[0]; + $iEndSequence = (int)$aOutput[0]; } } while ($iResult); // get the newest object from the diff file $sBatchEnd = 0; $iRet = 0; - exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet); + exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet); if ($iRet == 5) { echo "Diff file is empty. skipping import.\n"; if (!$aResult['import-osmosis-all']) { @@ -362,7 +388,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // Import the file $fCMDStartTime = time(); - echo $sCMDImport . "\n"; + echo $sCMDImport."\n"; unset($sJunk); $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv); if ($iErrorLevel) { @@ -375,8 +401,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; - $sSQL .= date('Y-m-d H:i:s') . "','import')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; + $sSQL .= date('Y-m-d H:i:s')."','import')"; var_Dump($sSQL); chksql($oDB->query($sSQL)); @@ -384,7 +410,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence"; var_Dump($sSQL); chksql($oDB->query($sSQL)); - echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; + echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; } // Index file @@ -402,21 +428,18 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; - $sSQL .= date('Y-m-d H:i:s') . "','index')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; + $sSQL .= date('Y-m-d H:i:s')."','index')"; var_Dump($sSQL); $oDB->query($sSQL); - echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; + echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; $sSQL = 'update import_status set indexed = true'; $oDB->query($sSQL); } $fDuration = time() - $fStartTime; - echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n"; - if (!$aResult['import-osmosis-all']) { - exit(0); - } - + echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n"; + if (!$aResult['import-osmosis-all']) exit(0); } }