From: Sarah Hoffmann <lonvia@denofr.de>
Date: Sun, 11 Jun 2017 07:05:24 +0000 (+0200)
Subject: improve update interval computation
X-Git-Tag: v3.0.0~4
X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/896421fe9f8dd9d49291417fe3d9389d71d967ca

improve update interval computation

Don't do anything if a downloaded diff is empty after all
(may be happening when an empty diff was published upstream).

Correctly compute the waiting interval before checking for new
data. As the interval is now computed based on the date of the
newest object in the database, the configured intervals need
to be adjusted slightly to take into account the time it takes
for the server to publish a diff.
---

diff --git a/settings/defaults.php b/settings/defaults.php
index 9f694c89..7c984a54 100644
--- a/settings/defaults.php
+++ b/settings/defaults.php
@@ -69,11 +69,18 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 @define('CONST_Tablespace_Aux_Data', false);
 @define('CONST_Tablespace_Aux_Index', false);
 
-// Replication settings
+//// Replication settings
+
+// Base URL of replication service
 @define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
-@define('CONST_Replication_Max_Diff_size', '30'); // MB of update data to download per batch
-@define('CONST_Replication_Update_Interval', '60');  // How often upstream publishes diffs
-@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
+
+// Maximum size in MB of data to download per batch
+@define('CONST_Replication_Max_Diff_size', '30');
+// How long until the service publishes the next diff
+// (relative to the age of data in the diff).
+@define('CONST_Replication_Update_Interval', '75');
+// How long to sleep when no update could be found
+@define('CONST_Replication_Recheck_Interval', '60');
 
 // Website settings
 @define('CONST_NoAccessControl', true);
diff --git a/utils/osm_file_date.py b/utils/osm_file_date.py
index 6418efb0..88d0a4c0 100755
--- a/utils/osm_file_date.py
+++ b/utils/osm_file_date.py
@@ -28,6 +28,7 @@ if __name__ == '__main__':
 
     h.apply_file(sys.argv[1])
 
-    print(h.filedate)
-
+    if h.filedate is None:
+        exit(5)
 
+    print(h.filedate)
diff --git a/utils/update.php b/utils/update.php
index f4716768..d8611715 100755
--- a/utils/update.php
+++ b/utils/update.php
@@ -241,9 +241,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
     while (true) {
         $fStartTime = time();
-        $iFileSize = 1001;
-
-        $aLastState = chksql($oDB->getRow('SELECT * FROM import_status'));
+        $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));
 
         if (!$aLastState['sequence_id']) {
             echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
@@ -257,7 +255,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
         if ($aLastState['indexed'] == 't') {
             // Sleep if the update interval has not yet been reached.
-            $fNextUpdate = $aLastState['lastimportdate'] + CONST_Replication_Update_Interval;
+            $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;
             if ($fNextUpdate > $fStartTime) {
                 $iSleepTime = $fNextUpdate - $fStartTime;
                 echo "Waiting for next update for $iSleepTime sec.";
@@ -284,6 +282,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 }
             } while ($iResult);
 
+            // get the newest object from the diff file
+            $sBatchEnd = 0;
+            $iRet = 0;
+            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
+            if ($iRet == 5) {
+                echo "Diff file is empty. skipping import.\n";
+                if (!$aResult['import-osmosis-all']) {
+                    exit(0);
+                } else {
+                    continue;
+                }
+            }
+            if ($iRet != 0) {
+                fail('Error getting date from diff file.');
+            }
+            $sBatchEnd = $sBatchEnd[0];
+
             // Import the file
             $fCMDStartTime = time();
             echo $sCMDImport."\n";
@@ -296,14 +311,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
-            // get the newest object from the diff file
-            $sBatchEnd = 0;
-            $iRet = 0;
-            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
-            if ($iRet != 0) {
-                fail('Error getting date from diff file.');
-            }
-            $sBatchEnd = $sBatchEnd[0];
             $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));