Merge pull request #1010 from lonvia/ignore-unicode-format-characters

[nominatim.git] / utils / update.php

diff --git a/utils/update.php b/utils/update.php
index 5a5d41fff8435f2c2c9cd50d16a004704458d3ba..cd907d7518dc5bdf05a761b3a87da73fa2265e14 100755 (executable)
--- a/utils/update.php
+++ b/utils/update.php
@@ -13,6 +13,7 @@ $aCMDOptions
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
 
    array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
 
    array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),

+   array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),

    array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
    array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
    array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
    array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
    array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
    array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),


@@ -33,6 +34,7 @@ $aCMDOptions
    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
 
    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
 
    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),

+   array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),

    array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
    array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);


@@ -54,7 +56,7 @@ if ($iCacheMemory + 500 > getTotalMemoryMB()) {
     echo "WARNING: resetting cache memory to $iCacheMemory\n";
 }
 $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
     echo "WARNING: resetting cache memory to $iCacheMemory\n";
 }
 $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];

-if (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
+if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {

     $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 }
 
     $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 }
 


@@ -97,6 +99,17 @@ if ($aResult['init-updates']) {
     echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";
 }
 
     echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";
 }
 

+if ($aResult['check-for-updates']) {
+    $aLastState = chksql($oDB->getRow('SELECT sequence_id FROM import_status'));
+
+    if (!$aLastState['sequence_id']) {
+        fail('Updates not set up. Please run ./utils/update.php --init-updates.');
+    }
+
+    system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet);
+    exit($iRet);
+}
+

 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
     // import diffs and files directly (e.g. from osmosis --rri)
     $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
     // import diffs and files directly (e.g. from osmosis --rri)
     $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];


@@ -129,25 +142,25 @@ $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-m
 $sContentURL = '';
 if (isset($aResult['import-node']) && $aResult['import-node']) {
     if ($bUseOSMApi) {
 $sContentURL = '';
 if (isset($aResult['import-node']) && $aResult['import-node']) {
     if ($bUseOSMApi) {

-        $sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];

     } else {
     } else {

-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';

     }
 }
 
 if (isset($aResult['import-way']) && $aResult['import-way']) {
     if ($bUseOSMApi) {
     }
 }
 
 if (isset($aResult['import-way']) && $aResult['import-way']) {
     if ($bUseOSMApi) {

-        $sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';

     } else {
     } else {

-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';

     }
 }
 
 if (isset($aResult['import-relation']) && $aResult['import-relation']) {
     if ($bUseOSMApi) {
     }
 }
 
 if (isset($aResult['import-relation']) && $aResult['import-relation']) {
     if ($bUseOSMApi) {

-        $sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
+        $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';

     } else {
     } else {

-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';

     }
 }
 
     }
 }
 


@@ -237,6 +250,12 @@ if ($aResult['deduplicate']) {
     }
 }
 
     }
 }
 

+if ($aResult['recompute-word-counts']) {
+    info('Recompute frequency of full-word search terms');
+    $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
+    runSQLScript($sTemplate, true, true);
+}
+

 if ($aResult['index']) {
     passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 }
 if ($aResult['index']) {
     passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 }


@@ -244,7 +263,8 @@ if ($aResult['index']) {
 if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
     //
     if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
 if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
     //
     if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {

-        fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
+        fail('Error: Update interval too low for download.geofabrik.de. ' .
+             "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");

     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';
     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';


@@ -281,7 +301,9 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 $iNextSeq = (int) $aLastState['sequence_id'];
                 unset($aOutput);
                 echo "$sCMDDownload -I $iNextSeq\n";
                 $iNextSeq = (int) $aLastState['sequence_id'];
                 unset($aOutput);
                 echo "$sCMDDownload -I $iNextSeq\n";

-                unlink($sImportFile);
+                if (file_exists($sImportFile)) {
+                    unlink($sImportFile);
+                }

                 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
 
                 if ($iResult == 3) {
                 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
 
                 if ($iResult == 3) {