don't log IPv6 IPs for the moment

[nominatim.git] / utils / update.php

diff --git a/utils/update.php b/utils/update.php
index ca700f0bd8a676628f377bd55f577429cf537cab..e35554888d4de0519571a6feca4afc1edddbaf8e 100755 (executable)
--- a/utils/update.php
+++ b/utils/update.php
@@ -7,7 +7,7 @@ ini_set('memory_limit', '800M');
 
 $aCMDOptions
 = array(
 
 $aCMDOptions
 = array(

-   "Import / update / index osm data",
+   'Import / update / index osm data',

    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),


@@ -33,11 +33,13 @@ $aCMDOptions
    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
 
    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
 
    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),

+   array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),

    array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
 if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
    array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
 if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;

+

 if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
 
 date_default_timezone_set('Etc/UTC');
 if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
 
 date_default_timezone_set('Etc/UTC');


@@ -64,7 +66,7 @@ if ($aResult['init-updates']) {
     if ($sBaseState === false) {
         echo "\nCannot find state.txt file at the configured replication URL.\n";
         echo "Does the URL point to a directory containing OSM update data?\n\n";
     if ($sBaseState === false) {
         echo "\nCannot find state.txt file at the configured replication URL.\n";
         echo "Does the URL point to a directory containing OSM update data?\n\n";

-        fail("replication URL not reachable.");
+        fail('replication URL not reachable.');

     }
     $sSetup = CONST_InstallPath.'/utils/setup.php';
     $iRet = -1;
     }
     $sSetup = CONST_InstallPath.'/utils/setup.php';
     $iRet = -1;


@@ -75,10 +77,9 @@ if ($aResult['init-updates']) {
 
     $sDatabaseDate = getDatabaseDate($oDB);
     if ($sDatabaseDate === false) {
 
     $sDatabaseDate = getDatabaseDate($oDB);
     if ($sDatabaseDate === false) {

-        fail("Cannot determine date of database.");
+        fail('Cannot determine date of database.');

     }
     }

-    $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
-                          strtotime($sDatabaseDate) - (3*60*60));
+    $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));

 
     // get the appropriate state id
     $aOutput = 0;
 
     // get the appropriate state id
     $aOutput = 0;


@@ -90,9 +91,9 @@ if ($aResult['init-updates']) {
 
     pg_query($oDB->connection, 'TRUNCATE import_status');
     $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
 
     pg_query($oDB->connection, 'TRUNCATE import_status');
     $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";

-    $sSQL .= $sDatabaseDate."',".$aOutput[0].", true)";
+    $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';

     if (!pg_query($oDB->connection, $sSQL)) {
     if (!pg_query($oDB->connection, $sSQL)) {

-        fail("Could not enter sequence into database.");
+        fail('Could not enter sequence into database.');

     }
 
     echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";
     }
 
     echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";


@@ -119,7 +120,7 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
 }
 
 if ($aResult['calculate-postcodes']) {
 }
 
 if ($aResult['calculate-postcodes']) {

-    info("Update postcodes centroids");
+    info('Update postcodes centroids');

     $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
     runSQLScript($sTemplate, true, true);
 }
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
     runSQLScript($sTemplate, true, true);
 }


@@ -130,25 +131,25 @@ $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-m
 $sContentURL = '';
 if (isset($aResult['import-node']) && $aResult['import-node']) {
     if ($bUseOSMApi) {
 $sContentURL = '';
 if (isset($aResult['import-node']) && $aResult['import-node']) {
     if ($bUseOSMApi) {

-        $sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];

     } else {
     } else {

-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';

     }
 }
 
 if (isset($aResult['import-way']) && $aResult['import-way']) {
     if ($bUseOSMApi) {
     }
 }
 
 if (isset($aResult['import-way']) && $aResult['import-way']) {
     if ($bUseOSMApi) {

-        $sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';

     } else {
     } else {

-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';

     }
 }
 
 if (isset($aResult['import-relation']) && $aResult['import-relation']) {
     if ($bUseOSMApi) {
     }
 }
 
 if (isset($aResult['import-relation']) && $aResult['import-relation']) {
     if ($bUseOSMApi) {

-        $sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
+        $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';

     } else {
     } else {

-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';

     }
 }
 
     }
 }
 


@@ -171,7 +172,7 @@ if ($aResult['deduplicate']) {
     $oDB =& getDB();
 
     if (getPostgresVersion($oDB) < 9.3) {
     $oDB =& getDB();
 
     if (getPostgresVersion($oDB) < 9.3) {

-        fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
+        fail('ERROR: deduplicate is only currently supported in postgresql 9.3');

     }
 
     $sSQL = 'select partition from country_name order by country_code';
     }
 
     $sSQL = 'select partition from country_name order by country_code';


@@ -180,7 +181,7 @@ if ($aResult['deduplicate']) {
 
     // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
     foreach ($aPartitions as $i => $sPartition) {
 
     // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
     foreach ($aPartitions as $i => $sPartition) {

-        $sSQL = "select count(*) from search_name_".$sPartition;
+        $sSQL = 'select count(*) from search_name_'.$sPartition;

         $nEntries = chksql($oDB->getOne($sSQL));
         if ($nEntries == 0) {
             unset($aPartitions[$i]);
         $nEntries = chksql($oDB->getOne($sSQL));
         if ($nEntries == 0) {
             unset($aPartitions[$i]);


@@ -188,14 +189,14 @@ if ($aResult['deduplicate']) {
     }
 
     $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '";
     }
 
     $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '";

-    $sSQL .= " and class is null and type is null and country_code is null";
-    $sSQL .= " group by word_token having count(*) > 1 order by word_token";
+    $sSQL .= ' and class is null and type is null and country_code is null';
+    $sSQL .= ' group by word_token having count(*) > 1 order by word_token';

     $aDuplicateTokens = chksql($oDB->getAll($sSQL));
     foreach ($aDuplicateTokens as $aToken) {
         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
     $aDuplicateTokens = chksql($oDB->getAll($sSQL));
     foreach ($aDuplicateTokens as $aToken) {
         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;

-        echo "Deduping ".$aToken['word_token']."\n";
-        $sSQL = "select word_id,";
-        $sSQL .= " (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num";
+        echo 'Deduping '.$aToken['word_token']."\n";
+        $sSQL = 'select word_id,';
+        $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';

         $sSQL .= " from word where word_token = '".$aToken['word_token'];
         $sSQL .= "' and class is null and type is null and country_code is null order by num desc";
         $aTokenSet = chksql($oDB->getAll($sSQL));
         $sSQL .= " from word where word_token = '".$aToken['word_token'];
         $sSQL .= "' and class is null and type is null and country_code is null order by num desc";
         $aTokenSet = chksql($oDB->getAll($sSQL));


@@ -204,40 +205,46 @@ if ($aResult['deduplicate']) {
         $iKeepID = $aKeep['word_id'];
 
         foreach ($aTokenSet as $aRemove) {
         $iKeepID = $aKeep['word_id'];
 
         foreach ($aTokenSet as $aRemove) {

-            $sSQL = "update search_name set";
-            $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
-            $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
-            $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
+            $sSQL = 'update search_name set';
+            $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),';
+            $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
+            $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';

             chksql($oDB->query($sSQL));
 
             chksql($oDB->query($sSQL));
 

-            $sSQL = "update search_name set";
-            $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
-            $sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
+            $sSQL = 'update search_name set';
+            $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
+            $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']';

             chksql($oDB->query($sSQL));
 
             chksql($oDB->query($sSQL));
 

-            $sSQL = "update location_area_country set";
-            $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
-            $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
+            $sSQL = 'update location_area_country set';
+            $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
+            $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';

             chksql($oDB->query($sSQL));
 
             foreach ($aPartitions as $sPartition) {
             chksql($oDB->query($sSQL));
 
             foreach ($aPartitions as $sPartition) {

-                $sSQL = "update search_name_".$sPartition." set";
-                $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
-                $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
+                $sSQL = 'update search_name_'.$sPartition.' set';
+                $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')';
+                $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';

                 chksql($oDB->query($sSQL));
 
                 chksql($oDB->query($sSQL));
 

-                $sSQL = "update location_area_country set";
-                $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
-                $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
+                $sSQL = 'update location_area_country set';
+                $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
+                $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';

                 chksql($oDB->query($sSQL));
             }
 
                 chksql($oDB->query($sSQL));
             }
 

-            $sSQL = "delete from word where word_id = ".$aRemove['word_id'];
+            $sSQL = 'delete from word where word_id = '.$aRemove['word_id'];

             chksql($oDB->query($sSQL));
         }
     }
 }
 
             chksql($oDB->query($sSQL));
         }
     }
 }
 

+if ($aResult['recompute-word-counts']) {
+    info('Recompute frequency of full-word search terms');
+    $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
+    runSQLScript($sTemplate, true, true);
+}
+

 if ($aResult['index']) {
     passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 }
 if ($aResult['index']) {
     passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 }


@@ -245,7 +252,7 @@ if ($aResult['index']) {
 if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
     //
     if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
 if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
     //
     if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {

-        fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
+        fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");

     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';
     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';


@@ -288,7 +295,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 if ($iResult == 3) {
                     echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
                     sleep(CONST_Replication_Recheck_Interval);
                 if ($iResult == 3) {
                     echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
                     sleep(CONST_Replication_Recheck_Interval);

-                } else if ($iResult != 0) {
+                } elseif ($iResult != 0) {

                     echo 'ERROR: updates failed.';
                     exit($iResult);
                 } else {
                     echo 'ERROR: updates failed.';
                     exit($iResult);
                 } else {


@@ -325,7 +332,11 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
 
             // write the update logs
             $iFileSize = filesize($sImportFile);

-            $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
+            $sSQL = 'INSERT INTO import_osmosis_log';
+            $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+            $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+            $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+            $sSQL .= date('Y-m-d H:i:s')."','import')";

             var_Dump($sSQL);
             chksql($oDB->query($sSQL));
 
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));
 


@@ -348,12 +359,16 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 exit($iErrorLevel);
             }
 
                 exit($iErrorLevel);
             }
 

-            $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+            $sSQL = 'INSERT INTO import_osmosis_log';
+            $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+            $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+            $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+            $sSQL .= date('Y-m-d H:i:s')."','index')";

             var_Dump($sSQL);
             $oDB->query($sSQL);
             echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
 
             var_Dump($sSQL);
             $oDB->query($sSQL);
             echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
 

-            $sSQL = "update import_status set indexed = true";
+            $sSQL = 'update import_status set indexed = true';

             $oDB->query($sSQL);
         }
 
             $oDB->query($sSQL);
         }
 


@@ -362,4 +377,3 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
         if (!$aResult['import-osmosis-all']) exit(0);
     }
 }
         if (!$aResult['import-osmosis-all']) exit(0);
     }
 }

-