X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/981649bf8366a2e67ee974816b9b0b288b259380..28ad09a20aa81c3ac1f5c50db4e9954057b4142c:/utils/update.php?ds=sidebyside diff --git a/utils/update.php b/utils/update.php index 3c6580dc..5c5b64ea 100755 --- a/utils/update.php +++ b/utils/update.php @@ -1,5 +1,6 @@ #!/usr/bin/php -Cq array("pipe", "w") // stderr ); $aPipes = array(); - $sCMD = $sBasePath.'/osm2pgsql/osm2pgsql -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' -'; + $sCMD = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' -'; echo $sCMD."\n"; $hProc = proc_open($sCMD, $aSpec, $aPipes); if (!is_resource($hProc)) @@ -298,13 +300,15 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sImportFile = CONST_BasePath.'/data/osmosischange.osc'; - $sOsmosisCMD = CONST_BasePath.'/osmosis-0.38/bin/osmosis'; + $sOsmosisCMD = CONST_Osmosis_Binary; $sOsmosisConfigDirectory = CONST_BasePath.'/settings'; $sDatabaseName = 'nominatim'; $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile; - $sCMDImport = $sBasePath.'/osm2pgsql/osm2pgsql -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' '.$sImportFile; - $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -t 15 -F '; -// $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -t 15 '; + $sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' '.$sImportFile; + $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -t '.$aResult['index-instances']; + if (!$aResult['no-npi']) { + $sCMDIndex .= '-F '; + } while(true) { $fStartTime = time(); @@ -323,10 +327,12 @@ $fCMDStartTime = time(); echo $sCMDDownload."\n"; exec($sCMDDownload, $sJunk, $iErrorLevel); - if ($iErrorLevel) + while ($iErrorLevel == 1) { echo "Error: $iErrorLevel\n"; - exit; + sleep(60); + echo 'Re-trying: '.$sCMDDownload."\n"; + exec($sCMDDownload, $sJunk, $iErrorLevel); } $iFileSize = filesize($sImportFile); $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory); @@ -362,7 +368,7 @@ $iFileID = $oDB->getOne('select nextval(\'file\')'); if (PEAR::isError($iFileID)) { - echo $oDB->getMessage()."\n"; + echo $iFileID->getMessage()."\n"; exit; } $sFileDir = CONST_BasePath.'/export/diff/'; @@ -371,30 +377,19 @@ if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true); $sThisIndexCmd = $sCMDIndex; - $sThisIndexCmd .= $sFileDir; - $sThisIndexCmd .= '/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT); - $sThisIndexCmd .= ".npi.out"; - echo "$sThisIndexCmd\n"; - - preg_match('#^([0-9]{4})-([0-9]{2})-([0-9]{2})#', $sBatchEnd, $aBatchMatch); - $sFileDir = CONST_BasePath.'/export/index/'; - $sFileDir .= $aBatchMatch[1].'/'.$aBatchMatch[2]; + if (!$aResult['no-npi']) { + $sThisIndexCmd .= $sFileDir; + $sThisIndexCmd .= '/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT); + $sThisIndexCmd .= ".npi.out"; - if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true); - file_put_contents($sFileDir.'/'.$aBatchMatch[3].'.idx', "$sBatchEnd\t$iFileID\n", FILE_APPEND); + preg_match('#^([0-9]{4})-([0-9]{2})-([0-9]{2})#', $sBatchEnd, $aBatchMatch); + $sFileDir = CONST_BasePath.'/export/index/'; + $sFileDir .= $aBatchMatch[1].'/'.$aBatchMatch[2]; - exec($sThisIndexCmd, $sJunk, $iErrorLevel); - if ($iErrorLevel) - { - echo "Error: $iErrorLevel\n"; - exit; + if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true); + file_put_contents($sFileDir.'/'.$aBatchMatch[3].'.idx', "$sBatchEnd\t$iFileID\n", FILE_APPEND); } - $sFileDir = CONST_BasePath.'/export/diff/'; - $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT); - $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT); - - $sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out"; echo "$sThisIndexCmd\n"; exec($sThisIndexCmd, $sJunk, $iErrorLevel); if ($iErrorLevel) @@ -403,8 +398,23 @@ exit; } - rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2", - $sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2"); + if (!$aResult['no-npi']) { + $sFileDir = CONST_BasePath.'/export/diff/'; + $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT); + $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT); + + $sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out"; + echo "$sThisIndexCmd\n"; + exec($sThisIndexCmd, $sJunk, $iErrorLevel); + if ($iErrorLevel) + { + echo "Error: $iErrorLevel\n"; + exit; + } + + rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2", + $sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2"); + } echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')"; @@ -418,7 +428,6 @@ echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n"; if (!$aResult['import-osmosis-all']) exit; // } - echo "Sleeping ".max(0,60-$fDuration)." seconds\n"; sleep(max(0,60-$fDuration)); } @@ -448,7 +457,7 @@ $sImportFile .= ".npi"; while(!file_exists($sImportFile) && !file_exists($sImportFile.'.bz2')) { - echo "sleep\n"; + echo "sleep (waiting for $sImportFile)\n"; sleep(10); } if (file_exists($sImportFile.'.bz2')) $sImportFile .= '.bz2';