Merge remote-tracking branch 'upstream/master'

[nominatim.git] / utils / update.php

diff --git a/utils/update.php b/utils/update.php
index ad6274825f074aab360a68414de039ce9e42d32c..3026dacb5b0ecb235f51a86881dedf5b1b309358 100755 (executable)
--- a/utils/update.php
+++ b/utils/update.php
@@ -15,10 +15,9 @@ $aCMDOptions
    array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
    array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
    array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
    array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
    array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
    array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),

-   array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),

    array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
 
    array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
 

-   array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
+   array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),

 
    array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
    array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
 
    array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
    array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),


@@ -34,15 +33,11 @@ $aCMDOptions
    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
 
    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
 
    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),

+   array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),

   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 

-if (!isset($aResult['index-instances'])) {
-    if (getLoadAverage() < 24)
-        $aResult['index-instances'] = 2;
-    else
-        $aResult['index-instances'] = 1;
-}
+if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;

 
 if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
 
 
 if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
 


@@ -65,6 +60,13 @@ if (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
 }
 
 if ($aResult['init-updates']) {
 }
 
 if ($aResult['init-updates']) {

+    // sanity check that the replication URL is correct
+    $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
+    if ($sBaseState === false) {
+        echo "\nCannot find state.txt file at the configured replication URL.\n";
+        echo "Does the URL point to a directory containing OSM update data?\n\n";
+        fail("replication URL not reachable.");
+    }

     $sSetup = CONST_InstallPath.'/utils/setup.php';
     $iRet = -1;
     passthru($sSetup.' --create-functions --enable-diff-updates', $iRet);
     $sSetup = CONST_InstallPath.'/utils/setup.php';
     $iRet = -1;
     passthru($sSetup.' --create-functions --enable-diff-updates', $iRet);


@@ -73,14 +75,16 @@ if ($aResult['init-updates']) {
     }
 
     $sDatabaseDate = getDatabaseDate($oDB);
     }
 
     $sDatabaseDate = getDatabaseDate($oDB);

-    $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
-                          strtotime($sDatabaseDate) - (3*60*60));
+    if ($sDatabaseDate === false) {
+        fail("Cannot determine date of database.");
+    }
+    $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));

 
     // get the appropriate state id
     $aOutput = 0;
 
     // get the appropriate state id
     $aOutput = 0;

-    exec(CONST_Pyosmium_Get_Changes.' -D '.$sWindBack.' --server '.CONST_Replication_Url,
-        $aOutput, $iRet);
-    if ($iRet != 0) {
+    $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
+    exec($sCmd, $aOutput, $iRet);
+    if ($iRet != 0 || $aOutput[0] == 'None') {

         fail('Error running pyosmium tools');
     }
 
         fail('Error running pyosmium tools');
     }
 


@@ -97,6 +101,7 @@ if ($aResult['init-updates']) {
 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
     // import diffs and files directly (e.g. from osmosis --rri)
     $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
     // import diffs and files directly (e.g. from osmosis --rri)
     $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];

+

     if (!file_exists($sNextFile)) {
         fail("Cannot open $sNextFile\n");
     }
     if (!file_exists($sNextFile)) {
         fail("Cannot open $sNextFile\n");
     }


@@ -113,6 +118,12 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
     // Don't update the import status - we don't know what this file contains
 }
 
     // Don't update the import status - we don't know what this file contains
 }
 

+if ($aResult['calculate-postcodes']) {
+    info("Update postcodes centroids");
+    $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
+    runSQLScript($sTemplate, true, true);
+}
+

 $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
 $bHaveDiff = false;
 $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
 $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
 $bHaveDiff = false;
 $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];


@@ -238,15 +249,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';
     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';

-    $sCMDDownload = CONST_Pyosmium_Get_Changes.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
+    $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;

     $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
     $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 
     while (true) {
         $fStartTime = time();
     $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
     $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 
     while (true) {
         $fStartTime = time();

-        $iFileSize = 1001;
-
-        $aLastState = chksql($oDB->getRow('SELECT * FROM import_status'));
+        $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));

 
         if (!$aLastState['sequence_id']) {
             echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
 
         if (!$aLastState['sequence_id']) {
             echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";


@@ -260,7 +269,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
         if ($aLastState['indexed'] == 't') {
             // Sleep if the update interval has not yet been reached.
 
         if ($aLastState['indexed'] == 't') {
             // Sleep if the update interval has not yet been reached.

-            $fNextUpdate = $aLastState['lastimportdate'] + CONST_Replication_Update_Interval;
+            $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;

             if ($fNextUpdate > $fStartTime) {
                 $iSleepTime = $fNextUpdate - $fStartTime;
                 echo "Waiting for next update for $iSleepTime sec.";
             if ($fNextUpdate > $fStartTime) {
                 $iSleepTime = $fNextUpdate - $fStartTime;
                 echo "Waiting for next update for $iSleepTime sec.";


@@ -268,18 +277,18 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             }
 
             // Download the next batch of changes.
             }
 
             // Download the next batch of changes.

-            unlink($sImportFile);

             do {
                 $fCMDStartTime = time();
             do {
                 $fCMDStartTime = time();

-                $iNextSeq = (int) $aLastState['sequence_id'] + 1;
+                $iNextSeq = (int) $aLastState['sequence_id'];

                 unset($aOutput);
                 echo "$sCMDDownload -I $iNextSeq\n";
                 unset($aOutput);
                 echo "$sCMDDownload -I $iNextSeq\n";

+                unlink($sImportFile);

                 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
 
                 if ($iResult == 3) {
                     echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
                     sleep(CONST_Replication_Recheck_Interval);
                 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
 
                 if ($iResult == 3) {
                     echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
                     sleep(CONST_Replication_Recheck_Interval);

-                } else if ($iResult != 0) {
+                } elseif ($iResult != 0) {

                     echo 'ERROR: updates failed.';
                     exit($iResult);
                 } else {
                     echo 'ERROR: updates failed.';
                     exit($iResult);
                 } else {


@@ -287,6 +296,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 }
             } while ($iResult);
 
                 }
             } while ($iResult);
 

+            // get the newest object from the diff file
+            $sBatchEnd = 0;
+            $iRet = 0;
+            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
+            if ($iRet == 5) {
+                echo "Diff file is empty. skipping import.\n";
+                if (!$aResult['import-osmosis-all']) {
+                    exit(0);
+                } else {
+                    continue;
+                }
+            }
+            if ($iRet != 0) {
+                fail('Error getting date from diff file.');
+            }
+            $sBatchEnd = $sBatchEnd[0];
+

             // Import the file
             $fCMDStartTime = time();
             echo $sCMDImport."\n";
             // Import the file
             $fCMDStartTime = time();
             echo $sCMDImport."\n";


@@ -299,8 +325,11 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
 
             // write the update logs
             $iFileSize = filesize($sImportFile);

-            $sBatchEnd = getDatabaseDate($oDB);
-            $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
+            $sSQL = 'INSERT INTO import_osmosis_log';
+            $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+            $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+            $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+            $sSQL .= date('Y-m-d H:i:s')."','import')";

             var_Dump($sSQL);
             chksql($oDB->query($sSQL));
 
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));
 


@@ -323,7 +352,11 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 exit($iErrorLevel);
             }
 
                 exit($iErrorLevel);
             }
 

-            $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+            $sSQL = 'INSERT INTO import_osmosis_log';
+            $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+            $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+            $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+            $sSQL .= date('Y-m-d H:i:s')."','index')";

             var_Dump($sSQL);
             $oDB->query($sSQL);
             echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
             var_Dump($sSQL);
             $oDB->query($sSQL);
             echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";


@@ -337,4 +370,3 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
         if (!$aResult['import-osmosis-all']) exit(0);
     }
 }
         if (!$aResult['import-osmosis-all']) exit(0);
     }
 }

-