X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/354487d7f475c0aecec484d077359f7ea44d4033..ac116980ac78b74dbc1e0ae7ff2d329d7e581ba5:/utils/update.php diff --git a/utils/update.php b/utils/update.php index 189a1825..ff8b0c00 100644 --- a/utils/update.php +++ b/utils/update.php @@ -1,9 +1,9 @@ addParams('--latlong') ->addParams('--append') ->addParams('--slim') + ->addParams('--with-forward-dependencies', 'false') + ->addParams('--log-progress', 'true') ->addParams('--number-processes', 1) ->addParams('--cache', $iCacheMemory) ->addParams('--output', 'gazetteer') @@ -99,11 +101,13 @@ if ($fPostgresVersion >= 11.0) { } -$oIndexCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py')) +$oIndexCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py')) ->addParams('--database', $aDSNInfo['database']) ->addParams('--port', $aDSNInfo['port']) ->addParams('--threads', $aResult['index-instances']); - +if (!$aResult['quiet']) { + $oIndexCmd->addParams('--verbose'); +} if ($aResult['verbose']) { $oIndexCmd->addParams('--verbose'); } @@ -151,7 +155,6 @@ if ($aResult['init-updates']) { 'enable-diff-updates' => true, 'verbose' => $aResult['verbose'] )); - $cSetup->connect(); $cSetup->createFunctions(); } @@ -192,7 +195,7 @@ if ($aResult['check-for-updates']) { fail('Updates not set up. Please run ./utils/update.php --init-updates.'); } - $oCmd = (new \Nominatim\Shell(CONST_BasePath.'/utils/check_server_for_updates.py')) + $oCmd = (new \Nominatim\Shell(CONST_BinDir.'/check_server_for_updates.py')) ->addParams(CONST_Replication_Url) ->addParams($aLastState['sequence_id']); $iRet = $oCmd->run(); @@ -222,11 +225,11 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { if ($aResult['calculate-postcodes']) { info('Update postcodes centroids'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql'); + $sTemplate = file_get_contents(CONST_DataDir.'/sql/update-postcodes.sql'); runSQLScript($sTemplate, true, true); } -$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc'; +$sTemporaryFile = CONST_InstallDir.'/osmosischange.osc'; $bHaveDiff = false; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; $sContentURL = ''; @@ -242,15 +245,15 @@ if (isset($aResult['import-way']) && $aResult['import-way']) { if ($bUseOSMApi) { $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');%3E;);out%20meta;'; } } if (isset($aResult['import-relation']) && $aResult['import-relation']) { if ($bUseOSMApi) { - $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full'; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=(rel(id:'.$aResult['import-relation'].');%3E;);out%20meta;'; } } @@ -271,89 +274,19 @@ if ($bHaveDiff) { } } -if ($aResult['deduplicate']) { - $oDB = new Nominatim\DB(); - $oDB->connect(); - - if ($oDB->getPostgresVersion() < 9.3) { - fail('ERROR: deduplicate is only currently supported in postgresql 9.3'); - } - - $sSQL = 'select partition from country_name order by country_code'; - $aPartitions = $oDB->getCol($sSQL); - $aPartitions[] = 0; - - // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates - foreach ($aPartitions as $i => $sPartition) { - $sSQL = 'select count(*) from search_name_'.$sPartition; - $nEntries = $oDB->getOne($sSQL); - if ($nEntries == 0) { - unset($aPartitions[$i]); - } - } - - $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '"; - $sSQL .= ' and class is null and type is null and country_code is null'; - $sSQL .= ' group by word_token having count(*) > 1 order by word_token'; - $aDuplicateTokens = $oDB->getAll($sSQL); - foreach ($aDuplicateTokens as $aToken) { - if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; - echo 'Deduping '.$aToken['word_token']."\n"; - $sSQL = 'select word_id,'; - $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num'; - $sSQL .= " from word where word_token = '".$aToken['word_token']; - $sSQL .= "' and class is null and type is null and country_code is null order by num desc"; - $aTokenSet = $oDB->getAll($sSQL); - - $aKeep = array_shift($aTokenSet); - $iKeepID = $aKeep['word_id']; - - foreach ($aTokenSet as $aRemove) { - $sSQL = 'update search_name set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - $sSQL = 'update search_name set'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - foreach ($aPartitions as $sPartition) { - $sSQL = 'update search_name_'.$sPartition.' set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - } - - $sSQL = 'delete from word where word_id = '.$aRemove['word_id']; - $oDB->exec($sSQL); - } - } -} - if ($aResult['recompute-word-counts']) { info('Recompute frequency of full-word search terms'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); + $sTemplate = file_get_contents(CONST_DataDir.'/sql/words_from_search_name.sql'); runSQLScript($sTemplate, true, true); } if ($aResult['index']) { $oCmd = (clone $oIndexCmd) - ->addParams('--minrank', $aResult['index-rank']); + ->addParams('--minrank', $aResult['index-rank'], '-b'); + $oCmd->run(); - // echo $oCmd->escapedCmd()."\n"; + $oCmd = (clone $oIndexCmd) + ->addParams('--minrank', $aResult['index-rank']); $oCmd->run(); $oDB->exec('update import_status set indexed = true'); @@ -389,7 +322,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { "Please check install documentation (https://nominatim.org/release-docs/latest/admin/Import-and-Update#setting-up-the-update-process)\n"); } - $sImportFile = CONST_InstallPath.'/osmosischange.osc'; + $sImportFile = CONST_InstallDir.'/osmosischange.osc'; $oCMDDownload = (new \Nominatim\Shell(CONST_Pyosmium_Binary)) ->addParams('--server', CONST_Replication_Url) @@ -448,7 +381,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // get the newest object from the diff file $sBatchEnd = 0; $iRet = 0; - $oCMD = new \Nominatim\Shell(CONST_BasePath.'/utils/osm_file_date.py', $sImportFile); + $oCMD = new \Nominatim\Shell(CONST_BinDir.'/osm_file_date.py', $sImportFile); exec($oCMD->escapedCmd(), $sBatchEnd, $iRet); if ($iRet == 5) { echo "Diff file is empty. skipping import.\n"; @@ -494,9 +427,18 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // Index file if (!$aResult['no-index']) { - $oThisIndexCmd = clone($oIndexCmd); $fCMDStartTime = time(); + $oThisIndexCmd = clone($oIndexCmd); + $oThisIndexCmd->addParams('-b'); + echo $oThisIndexCmd->escapedCmd()."\n"; + $iErrorLevel = $oThisIndexCmd->run(); + if ($iErrorLevel) { + echo "Error: $iErrorLevel\n"; + exit($iErrorLevel); + } + + $oThisIndexCmd = clone($oIndexCmd); echo $oThisIndexCmd->escapedCmd()."\n"; $iErrorLevel = $oThisIndexCmd->run(); if ($iErrorLevel) {