Merge remote-tracking branch 'upstream/master'

[nominatim.git] / utils / update.php

diff --git a/utils/update.php b/utils/update.php
index 77b76bc5ea64148ce6a1a74ed49c1d8b3d3cd035..1f036ff000bda0b8a2537c729358655eed12111b 100755 (executable)
--- a/utils/update.php
+++ b/utils/update.php
@@ -68,12 +68,15 @@ if ($aResult['init-updates']) {
     }
 
     $sDatabaseDate = getDatabaseDate($oDB);
     }
 
     $sDatabaseDate = getDatabaseDate($oDB);

+    if ($sDatabaseDate === false) {
+        fail("Cannot determine date of database.");
+    }

     $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
                           strtotime($sDatabaseDate) - (3*60*60));
 
     // get the appropriate state id
     $aOutput = 0;
     $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
                           strtotime($sDatabaseDate) - (3*60*60));
 
     // get the appropriate state id
     $aOutput = 0;

-    exec(CONST_Pyosmium_Get_Changes.' -D '.$sWindBack.' --server '.CONST_Replication_Url,
+    exec(CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url,

         $aOutput, $iRet);
     if ($iRet != 0) {
         fail('Error running pyosmium tools');
         $aOutput, $iRet);
     if ($iRet != 0) {
         fail('Error running pyosmium tools');


@@ -233,15 +236,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';
     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';

-    $sCMDDownload = CONST_Pyosmium_Get_Changes.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
+    $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;

     $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
     $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 
     while (true) {
         $fStartTime = time();
     $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
     $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 
     while (true) {
         $fStartTime = time();

-        $iFileSize = 1001;
-
-        $aLastState = chksql($oDB->getRow('SELECT * FROM import_status'));
+        $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));

 
         if (!$aLastState['sequence_id']) {
             echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
 
         if (!$aLastState['sequence_id']) {
             echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";


@@ -255,7 +256,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
         if ($aLastState['indexed'] == 't') {
             // Sleep if the update interval has not yet been reached.
 
         if ($aLastState['indexed'] == 't') {
             // Sleep if the update interval has not yet been reached.

-            $fNextUpdate = $aLastState['lastimportdate'] + CONST_Replication_Update_Interval;
+            $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;

             if ($fNextUpdate > $fStartTime) {
                 $iSleepTime = $fNextUpdate - $fStartTime;
                 echo "Waiting for next update for $iSleepTime sec.";
             if ($fNextUpdate > $fStartTime) {
                 $iSleepTime = $fNextUpdate - $fStartTime;
                 echo "Waiting for next update for $iSleepTime sec.";


@@ -263,12 +264,12 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             }
 
             // Download the next batch of changes.
             }
 
             // Download the next batch of changes.

-            unlink($sImportFile);

             do {
                 $fCMDStartTime = time();
             do {
                 $fCMDStartTime = time();

-                $iNextSeq = (int) $aLastState['sequence_id'] + 1;
+                $iNextSeq = (int) $aLastState['sequence_id'];

                 unset($aOutput);
                 echo "$sCMDDownload -I $iNextSeq\n";
                 unset($aOutput);
                 echo "$sCMDDownload -I $iNextSeq\n";

+                unlink($sImportFile);

                 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
 
                 if ($iResult == 3) {
                 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
 
                 if ($iResult == 3) {


@@ -282,6 +283,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 }
             } while ($iResult);
 
                 }
             } while ($iResult);
 

+            // get the newest object from the diff file
+            $sBatchEnd = 0;
+            $iRet = 0;
+            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
+            if ($iRet == 5) {
+                echo "Diff file is empty. skipping import.\n";
+                if (!$aResult['import-osmosis-all']) {
+                    exit(0);
+                } else {
+                    continue;
+                }
+            }
+            if ($iRet != 0) {
+                fail('Error getting date from diff file.');
+            }
+            $sBatchEnd = $sBatchEnd[0];
+

             // Import the file
             $fCMDStartTime = time();
             echo $sCMDImport."\n";
             // Import the file
             $fCMDStartTime = time();
             echo $sCMDImport."\n";


@@ -294,7 +312,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
 
             // write the update logs
             $iFileSize = filesize($sImportFile);

-            $sBatchEnd = getDatabaseDate($oDB);

             $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));
             $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));