X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/89a365787f2e14d531dfeeeb0bf3bcb1d2872af8..8d221e2c8263133b9784803dc7f61701ae872254:/utils/update.php diff --git a/utils/update.php b/utils/update.php index dbf158e5..f8bbfb15 100755 --- a/utils/update.php +++ b/utils/update.php @@ -15,6 +15,8 @@ array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'), array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'), + array('no-npi', '', 0, 1, 0, 0, 'bool', 'Do not write npi index files'), + array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), array('import-npi-all', '', 0, 1, 0, 0, 'bool', 'Import npi pre-indexed files'), @@ -106,7 +108,7 @@ if (($aResult['import-hourly'] || $aResult['import-daily']) && file_exists($sNextFile)) { // Import the file - $sCMD = $sBasePath.'/osm2pgsql/osm2pgsql -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' '.$sNextFile; + $sCMD = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' '.$sNextFile; echo $sCMD."\n"; exec($sCMD, $sJunk, $iErrorLevel); @@ -147,6 +149,8 @@ // Hack into a modify request $sModifyXML = str_replace('', '', $sModifyXML); + $sModifyXML = str_replace('', + '', $sModifyXML); $sModifyXML = str_replace('', '', $sModifyXML); // Outputing this is too verbose @@ -159,7 +163,7 @@ 2 => array("pipe", "w") // stderr ); $aPipes = array(); - $sCMD = $sBasePath.'/osm2pgsql/osm2pgsql -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' -'; + $sCMD = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' -'; echo $sCMD."\n"; $hProc = proc_open($sCMD, $aSpec, $aPipes); if (!is_resource($hProc)) @@ -299,13 +303,15 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sImportFile = CONST_BasePath.'/data/osmosischange.osc'; - $sOsmosisCMD = CONST_BasePath.'/osmosis-0.38/bin/osmosis'; + $sOsmosisCMD = CONST_Osmosis_Binary; $sOsmosisConfigDirectory = CONST_BasePath.'/settings'; $sDatabaseName = 'nominatim'; $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile; - $sCMDImport = $sBasePath.'/osm2pgsql/osm2pgsql -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' '.$sImportFile; - $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -t 15 -F '; -// $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -t 15 '; + $sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' '.$sImportFile; + $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -t '.$aResult['index-instances']; + if (!$aResult['no-npi']) { + $sCMDIndex .= '-F '; + } while(true) { $fStartTime = time(); @@ -357,10 +363,15 @@ // Archive for debug? unlink($sImportFile); +// } - $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory); + $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory); + + // Index file + $sThisIndexCmd = $sCMDIndex; - // Index file + if (!$aResult['no-npi']) + { $fCMDStartTime = time(); $iFileID = $oDB->getOne('select nextval(\'file\')'); if (PEAR::isError($iFileID)) @@ -373,11 +384,9 @@ $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT); if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true); - $sThisIndexCmd = $sCMDIndex; $sThisIndexCmd .= $sFileDir; $sThisIndexCmd .= '/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT); $sThisIndexCmd .= ".npi.out"; - echo "$sThisIndexCmd\n"; preg_match('#^([0-9]{4})-([0-9]{2})-([0-9]{2})#', $sBatchEnd, $aBatchMatch); $sFileDir = CONST_BasePath.'/export/index/'; @@ -385,7 +394,11 @@ if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true); file_put_contents($sFileDir.'/'.$aBatchMatch[3].'.idx', "$sBatchEnd\t$iFileID\n", FILE_APPEND); + } + if (!$aResult['no-index']) + { + echo "$sThisIndexCmd\n"; exec($sThisIndexCmd, $sJunk, $iErrorLevel); if ($iErrorLevel) { @@ -393,38 +406,41 @@ exit; } - $sFileDir = CONST_BasePath.'/export/diff/'; - $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT); - $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT); - - $sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out"; - echo "$sThisIndexCmd\n"; - exec($sThisIndexCmd, $sJunk, $iErrorLevel); - if ($iErrorLevel) + if (!$aResult['no-npi']) { - echo "Error: $iErrorLevel\n"; - exit; - } + $sFileDir = CONST_BasePath.'/export/diff/'; + $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT); + $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT); + + $sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out"; + echo "$sThisIndexCmd\n"; + exec($sThisIndexCmd, $sJunk, $iErrorLevel); + if ($iErrorLevel) + { + echo "Error: $iErrorLevel\n"; + exit; + } - rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2", - $sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2"); + rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2", + $sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2"); + } + } - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; - $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')"; - $oDB->query($sSQL); + echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')"; + $oDB->query($sSQL); - $sSQL = "update import_status set lastimportdate = '$sBatchEnd'"; - $oDB->query($sSQL); + $sSQL = "update import_status set lastimportdate = '$sBatchEnd'"; + $oDB->query($sSQL); + $fDuration = time() - $fStartTime; + echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n"; + if (!$aResult['import-osmosis-all']) exit; - $fDuration = time() - $fStartTime; - echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n"; - if (!$aResult['import-osmosis-all']) exit; -// } echo "Sleeping ".max(0,60-$fDuration)." seconds\n"; sleep(max(0,60-$fDuration)); } - + } if ($aResult['import-npi-all'])