array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
- array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
- array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
+ array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
+ array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
);
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
+
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
date_default_timezone_set('Etc/UTC');
}
if ($aResult['init-updates']) {
+ // sanity check that the replication URL is correct
+ $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
+ if ($sBaseState === false) {
+ echo "\nCannot find state.txt file at the configured replication URL.\n";
+ echo "Does the URL point to a directory containing OSM update data?\n\n";
+ fail("replication URL not reachable.");
+ }
$sSetup = CONST_InstallPath.'/utils/setup.php';
$iRet = -1;
passthru($sSetup.' --create-functions --enable-diff-updates', $iRet);
}
$sDatabaseDate = getDatabaseDate($oDB);
- $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
- strtotime($sDatabaseDate) - (3*60*60));
+ if ($sDatabaseDate === false) {
+ fail("Cannot determine date of database.");
+ }
+ $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
// get the appropriate state id
$aOutput = 0;
- exec(CONST_Pyosmium_Get_Changes.' -D '.$sWindBack.' --server '.CONST_Replication_Url,
- $aOutput, $iRet);
- if ($iRet != 0) {
+ $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
+ exec($sCmd, $aOutput, $iRet);
+ if ($iRet != 0 || $aOutput[0] == 'None') {
fail('Error running pyosmium tools');
}
if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
// import diffs and files directly (e.g. from osmosis --rri)
$sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
+
if (!file_exists($sNextFile)) {
fail("Cannot open $sNextFile\n");
}
// Don't update the import status - we don't know what this file contains
}
+if ($aResult['calculate-postcodes']) {
+ info("Update postcodes centroids");
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
+ runSQLScript($sTemplate, true, true);
+}
+
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
$bHaveDiff = false;
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
}
$sImportFile = CONST_InstallPath.'/osmosischange.osc';
- $sCMDDownload = CONST_Pyosmium_Get_Changes.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
+ $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
while (true) {
$fStartTime = time();
- $iFileSize = 1001;
-
- $aLastState = chksql($oDB->getRow('SELECT * FROM import_status'));
+ $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));
if (!$aLastState['sequence_id']) {
echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
if ($aLastState['indexed'] == 't') {
// Sleep if the update interval has not yet been reached.
- $fNextUpdate = $aLastState['lastimportdate'] + CONST_Replication_Update_Interval;
+ $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;
if ($fNextUpdate > $fStartTime) {
$iSleepTime = $fNextUpdate - $fStartTime;
echo "Waiting for next update for $iSleepTime sec.";
}
// Download the next batch of changes.
- unlink($sImportFile);
do {
$fCMDStartTime = time();
- $iNextSeq = (int) $aLastState['sequence_id'] + 1;
+ $iNextSeq = (int) $aLastState['sequence_id'];
unset($aOutput);
echo "$sCMDDownload -I $iNextSeq\n";
+ unlink($sImportFile);
exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
if ($iResult == 3) {
echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
sleep(CONST_Replication_Recheck_Interval);
- } else if ($iResult != 0) {
+ } elseif ($iResult != 0) {
echo 'ERROR: updates failed.';
exit($iResult);
} else {
}
} while ($iResult);
+ // get the newest object from the diff file
+ $sBatchEnd = 0;
+ $iRet = 0;
+ exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
+ if ($iRet == 5) {
+ echo "Diff file is empty. skipping import.\n";
+ if (!$aResult['import-osmosis-all']) {
+ exit(0);
+ } else {
+ continue;
+ }
+ }
+ if ($iRet != 0) {
+ fail('Error getting date from diff file.');
+ }
+ $sBatchEnd = $sBatchEnd[0];
+
// Import the file
$fCMDStartTime = time();
echo $sCMDImport."\n";
// write the update logs
$iFileSize = filesize($sImportFile);
- $sBatchEnd = getDatabaseDate($oDB);
- $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
+ $sSQL = 'INSERT INTO import_osmosis_log';
+ $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+ $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+ $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+ $sSQL .= date('Y-m-d H:i:s')."','import')";
var_Dump($sSQL);
chksql($oDB->query($sSQL));
exit($iErrorLevel);
}
- $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+ $sSQL = 'INSERT INTO import_osmosis_log';
+ $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+ $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+ $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+ $sSQL .= date('Y-m-d H:i:s')."','index')";
var_Dump($sSQL);
$oDB->query($sSQL);
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
if (!$aResult['import-osmosis-all']) exit(0);
}
}
-
projects / nominatim.git / blobdiff