From: IrlJidel <donal.diamond@gmail.com>
Date: Tue, 26 Mar 2013 14:49:35 +0000 (+0000)
Subject: fix typo for baseUrl when configuring configuration.txt
X-Git-Tag: v2.2.0~98
X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/3eea62363d50b77d9ac94254bc2f5b426c29201a

fix typo for baseUrl when configuring configuration.txt

use osmosis --read-replication-lag to determine if there are changes before trying to process updates, useful when we are tracking hourly or daily replication updates

set CONST_Replication_Recheck_Interval to 60

skip lag check if CONST_Replication_Update_Interval > 60, for minutelies there's always new diffs to process

use tabs for indent

change sleep for non-minutely updates so that we dont drift tto much or poll excessively

unset $aReplicationLag before each exec attempt

unset $aReplicationLag inside while loop
---

diff --git a/settings/settings.php b/settings/settings.php
index 10f77ee4..771cf80f 100644
--- a/settings/settings.php
+++ b/settings/settings.php
@@ -16,8 +16,12 @@
 	@define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-'.CONST_Postgis_Version);
 	@define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
 	@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
+
+	// Replication settings
 	@define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
 	@define('CONST_Replication_MaxInterval', '3600');
+	@define('CONST_Replication_Update_Interval', '60');  // How often upstream publishes diffs
+	@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
 
 	// Connection buckets to rate limit people being nasty
 	@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
diff --git a/utils/setup.php b/utils/setup.php
index d404f4df..5ec22dd0 100755
--- a/utils/setup.php
+++ b/utils/setup.php
@@ -505,7 +505,7 @@
 			
 			passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 			// update osmosis configuration.txt with our settings
-			passthru("sed -i 's!baseUrl=.*!baseUrL=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
+			passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
 			passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
 		}
 
diff --git a/utils/update.php b/utils/update.php
index 9c50db71..76d6f858 100755
--- a/utils/update.php
+++ b/utils/update.php
@@ -353,6 +353,7 @@
 		$sOsmosisCMD = CONST_Osmosis_Binary;
 		$sOsmosisConfigDirectory = CONST_BasePath.'/settings';
 		$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
+		$sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
 		$sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sImportFile;
 		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'];
 		if (!$aResult['no-npi']) {
@@ -372,7 +373,30 @@
 //			{
 				if (!file_exists($sImportFile))
 				{
-					// Use osmosis to download the file
+					// First check if there are new updates published (except for minutelies - there's always new diffs to process)
+					if ( CONST_Replication_Update_Interval > 60 )
+					{
+
+						unset($aReplicationLag);
+						exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+						while ($iErrorLevel == 1 || $aReplicationLag[0] < 1)
+						{
+							if ($iErrorLevel)
+							{
+								echo "Error: $iErrorLevel. ";
+								echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
+							}
+							else
+							{
+								echo ".";
+							}
+							sleep(CONST_Replication_Recheck_Interval);
+							unset($aReplicationLag);
+							exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+						}
+						// There are new replication files - use osmosis to download the file
+						echo "\nReplication Delay is ".$aReplicationLag[0]."\n";
+					}
 					$fCMDStartTime = time();
 					echo $sCMDDownload."\n";
 					exec($sCMDDownload, $sJunk, $iErrorLevel);
@@ -483,8 +507,16 @@
 			echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
 			if (!$aResult['import-osmosis-all']) exit;
 
-			echo "Sleeping ".max(0,60-$fDuration)." seconds\n";
-			sleep(max(0,60-$fDuration));
+			if ( CONST_Replication_Update_Interval > 60 )
+			{
+				$iSleep = round(CONST_Replication_Update_Interval*0.8);
+			}
+			else
+			{
+				$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
+			}
+			echo "Sleeping $iSleep seconds\n";
+			sleep($iSleep);
 		}
 
 	}