fix typo for baseUrl when configuring configuration.txt

use osmosis --read-replication-lag to determine if there are changes before trying to process updates, useful when we are tracking hourly or daily replication updates

set CONST_Replication_Recheck_Interval to 60

skip lag check if CONST_Replication_Update_Interval > 60, for minutelies there's always new diffs to process

use tabs for indent

change sleep for non-minutely updates so that we dont drift tto much or poll excessively

unset $aReplicationLag before each exec attempt

unset $aReplicationLag inside while loop

diff --git a/settings/settings.php b/settings/settings.php
index 10f77ee4d988cd8d3a20c792d18a064fcabbae33..771cf80fa6e5f723f340a29f0c3ce2b1df335845 100644 (file)
--- a/settings/settings.php
+++ b/settings/settings.php
@@ -16,8 +16,12 @@
        @define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-'.CONST_Postgis_Version);
        @define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
        @define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
+
+       // Replication settings
        @define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
        @define('CONST_Replication_MaxInterval', '3600');
+       @define('CONST_Replication_Update_Interval', '60');  // How often upstream publishes diffs
+       @define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
 
        // Connection buckets to rate limit people being nasty
        @define('CONST_ConnectionBucket_MemcacheServerAddress', false);

diff --git a/utils/setup.php b/utils/setup.php
index d404f4df85057e422440d77402a4bd446cd20222..5ec22dd0947023aaca96d8f5f9cbeac1f3f97254 100755 (executable)
--- a/utils/setup.php
+++ b/utils/setup.php
@@ -505,7 +505,7 @@
                        
                        passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
                        // update osmosis configuration.txt with our settings
-                       passthru("sed -i 's!baseUrl=.*!baseUrL=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
+                       passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
                        passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
                }
 

diff --git a/utils/update.php b/utils/update.php
index 9c50db71dc08f5b0e3e25bfc0188dfb4c074531e..76d6f8582c2e1e9646845890b1d451b5ff8c6527 100755 (executable)
--- a/utils/update.php
+++ b/utils/update.php
@@ -353,6 +353,7 @@
                $sOsmosisCMD = CONST_Osmosis_Binary;
                $sOsmosisConfigDirectory = CONST_BasePath.'/settings';
                $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
+               $sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
                $sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sImportFile;
                $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'];
                if (!$aResult['no-npi']) {
@@ -372,7 +373,30 @@
 //                     {
                                if (!file_exists($sImportFile))
                                {
-                                       // Use osmosis to download the file
+                                       // First check if there are new updates published (except for minutelies - there's always new diffs to process)
+                                       if ( CONST_Replication_Update_Interval > 60 )
+                                       {
+
+                                               unset($aReplicationLag);
+                                               exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+                                               while ($iErrorLevel == 1 || $aReplicationLag[0] < 1)
+                                               {
+                                                       if ($iErrorLevel)
+                                                       {
+                                                               echo "Error: $iErrorLevel. ";
+                                                               echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
+                                                       }
+                                                       else
+                                                       {
+                                                               echo ".";
+                                                       }
+                                                       sleep(CONST_Replication_Recheck_Interval);
+                                                       unset($aReplicationLag);
+                                                       exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+                                               }
+                                               // There are new replication files - use osmosis to download the file
+                                               echo "\nReplication Delay is ".$aReplicationLag[0]."\n";
+                                       }
                                        $fCMDStartTime = time();
                                        echo $sCMDDownload."\n";
                                        exec($sCMDDownload, $sJunk, $iErrorLevel);
@@ -483,8 +507,16 @@
                        echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
                        if (!$aResult['import-osmosis-all']) exit;
 
-                       echo "Sleeping ".max(0,60-$fDuration)." seconds\n";
-                       sleep(max(0,60-$fDuration));
+                       if ( CONST_Replication_Update_Interval > 60 )
+                       {
+                               $iSleep = round(CONST_Replication_Update_Interval*0.8);
+                       }
+                       else
+                       {
+                               $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
+                       }
+                       echo "Sleeping $iSleep seconds\n";
+                       sleep($iSleep);
                }
 
        }