improve update interval computation

Don't do anything if a downloaded diff is empty after all
(may be happening when an empty diff was published upstream).

Correctly compute the waiting interval before checking for new
data. As the interval is now computed based on the date of the
newest object in the database, the configured intervals need
to be adjusted slightly to take into account the time it takes
for the server to publish a diff.

diff --git a/settings/defaults.php b/settings/defaults.php
index 9f694c89fb24699a62ee1d05c09b20b57c98f59a..7c984a542d3770ecaf09534c42a4c1325bf44d35 100644 (file)
--- a/settings/defaults.php
+++ b/settings/defaults.php
@@ -69,11 +69,18 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 @define('CONST_Tablespace_Aux_Data', false);
 @define('CONST_Tablespace_Aux_Index', false);
 
 @define('CONST_Tablespace_Aux_Data', false);
 @define('CONST_Tablespace_Aux_Index', false);
 

-// Replication settings
+//// Replication settings
+
+// Base URL of replication service

 @define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
 @define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');

-@define('CONST_Replication_Max_Diff_size', '30'); // MB of update data to download per batch
-@define('CONST_Replication_Update_Interval', '60');  // How often upstream publishes diffs
-@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
+
+// Maximum size in MB of data to download per batch
+@define('CONST_Replication_Max_Diff_size', '30');
+// How long until the service publishes the next diff
+// (relative to the age of data in the diff).
+@define('CONST_Replication_Update_Interval', '75');
+// How long to sleep when no update could be found
+@define('CONST_Replication_Recheck_Interval', '60');

 
 // Website settings
 @define('CONST_NoAccessControl', true);
 
 // Website settings
 @define('CONST_NoAccessControl', true);

diff --git a/utils/osm_file_date.py b/utils/osm_file_date.py
index 6418efb0c51b60280ff8aace0ea89ae4272dc242..88d0a4c0a6a46a4acb185f06b1fc8d70897af224 100755 (executable)
--- a/utils/osm_file_date.py
+++ b/utils/osm_file_date.py
@@ -28,6 +28,7 @@ if __name__ == '__main__':
 
     h.apply_file(sys.argv[1])
 
 
     h.apply_file(sys.argv[1])
 

-    print(h.filedate)
-
+    if h.filedate is None:
+        exit(5)

 
 

+    print(h.filedate)

diff --git a/utils/update.php b/utils/update.php
index f47167687a756e1200d628cd763411baa06eb6b9..d86117153c2efa4071fc02c213a96d6303eb648b 100755 (executable)
--- a/utils/update.php
+++ b/utils/update.php
@@ -241,9 +241,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
     while (true) {
         $fStartTime = time();
 
     while (true) {
         $fStartTime = time();

-        $iFileSize = 1001;
-
-        $aLastState = chksql($oDB->getRow('SELECT * FROM import_status'));
+        $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));

 
         if (!$aLastState['sequence_id']) {
             echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
 
         if (!$aLastState['sequence_id']) {
             echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";


@@ -257,7 +255,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
         if ($aLastState['indexed'] == 't') {
             // Sleep if the update interval has not yet been reached.
 
         if ($aLastState['indexed'] == 't') {
             // Sleep if the update interval has not yet been reached.

-            $fNextUpdate = $aLastState['lastimportdate'] + CONST_Replication_Update_Interval;
+            $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;

             if ($fNextUpdate > $fStartTime) {
                 $iSleepTime = $fNextUpdate - $fStartTime;
                 echo "Waiting for next update for $iSleepTime sec.";
             if ($fNextUpdate > $fStartTime) {
                 $iSleepTime = $fNextUpdate - $fStartTime;
                 echo "Waiting for next update for $iSleepTime sec.";


@@ -284,6 +282,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 }
             } while ($iResult);
 
                 }
             } while ($iResult);
 

+            // get the newest object from the diff file
+            $sBatchEnd = 0;
+            $iRet = 0;
+            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
+            if ($iRet == 5) {
+                echo "Diff file is empty. skipping import.\n";
+                if (!$aResult['import-osmosis-all']) {
+                    exit(0);
+                } else {
+                    continue;
+                }
+            }
+            if ($iRet != 0) {
+                fail('Error getting date from diff file.');
+            }
+            $sBatchEnd = $sBatchEnd[0];
+

             // Import the file
             $fCMDStartTime = time();
             echo $sCMDImport."\n";
             // Import the file
             $fCMDStartTime = time();
             echo $sCMDImport."\n";


@@ -296,14 +311,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
 
             // write the update logs
             $iFileSize = filesize($sImportFile);

-            // get the newest object from the diff file
-            $sBatchEnd = 0;
-            $iRet = 0;
-            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
-            if ($iRet != 0) {
-                fail('Error getting date from diff file.');
-            }
-            $sBatchEnd = $sBatchEnd[0];

             $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));
             $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));