Merge remote-tracking branch 'upstream/master'

[nominatim.git] / utils / update.php

diff --git a/utils/update.php b/utils/update.php
index 04006fdeab4a39efa06ffe6373129969f73d354a..8137584c71cc2540504fa5755eb11608ff8fe6cb 100755 (executable)
--- a/utils/update.php
+++ b/utils/update.php
@@ -166,12 +166,17 @@ if ($aResult['deduplicate']) {
     $aPartitions = chksql($oDB->getCol($sSQL));
     $aPartitions[] = 0;
 
     $aPartitions = chksql($oDB->getCol($sSQL));
     $aPartitions[] = 0;
 

-    $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
+    $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '";
+    $sSQL .= " and class is null and type is null and country_code is null";
+    $sSQL .= " group by word_token having count(*) > 1 order by word_token";

     $aDuplicateTokens = chksql($oDB->getAll($sSQL));
     foreach ($aDuplicateTokens as $aToken) {
         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
         echo "Deduping ".$aToken['word_token']."\n";
     $aDuplicateTokens = chksql($oDB->getAll($sSQL));
     foreach ($aDuplicateTokens as $aToken) {
         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
         echo "Deduping ".$aToken['word_token']."\n";

-        $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
+        $sSQL = "select word_id,";
+        $sSQL .= " (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num";
+        $sSQL .= " from word where word_token = '".$aToken['word_token'];
+        $sSQL .= "' and class is null and type is null and country_code is null order by num desc";

         $aTokenSet = chksql($oDB->getAll($sSQL));
 
         $aKeep = array_shift($aTokenSet);
         $aTokenSet = chksql($oDB->getAll($sSQL));
 
         $aKeep = array_shift($aTokenSet);


@@ -237,7 +242,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             // First check if there are new updates published (except for minutelies - there's always new diffs to process)
             if (CONST_Replication_Update_Interval > 60) {
                 unset($aReplicationLag);
             // First check if there are new updates published (except for minutelies - there's always new diffs to process)
             if (CONST_Replication_Update_Interval > 60) {
                 unset($aReplicationLag);

-                exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+                exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);

                 while ($iErrorLevel > 0 || $aReplicationLag[0] < 1) {
                     if ($iErrorLevel) {
                         echo "Error: $iErrorLevel. ";
                 while ($iErrorLevel > 0 || $aReplicationLag[0] < 1) {
                     if ($iErrorLevel) {
                         echo "Error: $iErrorLevel. ";


@@ -247,7 +252,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                     }
                     sleep(CONST_Replication_Recheck_Interval);
                     unset($aReplicationLag);
                     }
                     sleep(CONST_Replication_Recheck_Interval);
                     unset($aReplicationLag);

-                    exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+                    exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);

                 }
                 // There are new replication files - use osmosis to download the file
                 echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
                 }
                 // There are new replication files - use osmosis to download the file
                 echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";


@@ -264,10 +269,10 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             }
             $iFileSize = filesize($sImportFile);
             $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
             }
             $iFileSize = filesize($sImportFile);
             $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);

-            $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
+            $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";

             var_Dump($sSQL);
             $oDB->query($sSQL);
             var_Dump($sSQL);
             $oDB->query($sSQL);

-            echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
+            echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";

         }
 
         $iFileSize = filesize($sImportFile);
         }
 
         $iFileSize = filesize($sImportFile);


@@ -281,10 +286,10 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             echo "Error: $iErrorLevel\n";
             exit($iErrorLevel);
         }
             echo "Error: $iErrorLevel\n";
             exit($iErrorLevel);
         }

-        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
+        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";

         var_Dump($sSQL);
         $oDB->query($sSQL);
         var_Dump($sSQL);
         $oDB->query($sSQL);

-        echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
+        echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";

 
         // Archive for debug?
         unlink($sImportFile);
 
         // Archive for debug?
         unlink($sImportFile);


@@ -293,6 +298,15 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
         // Index file
         $sThisIndexCmd = $sCMDIndex;
 
         // Index file
         $sThisIndexCmd = $sCMDIndex;

+        if (!isset($aResult['index-instances'])) {
+            if (getLoadAverage() < 24)
+                $iIndexInstances = 2;
+            else
+                $iIndexInstances = 1;
+        } else
+            $iIndexInstances = $aResult['index-instances'];
+
+        $sThisIndexCmd = $sCMDIndex.' -t '.$iIndexInstances;

         $fCMDStartTime = time();
 
         if (!$aResult['no-index']) {
         $fCMDStartTime = time();
 
         if (!$aResult['no-index']) {


@@ -304,31 +318,32 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             }
         }
 
             }
         }
 

-        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";

         var_Dump($sSQL);
         $oDB->query($sSQL);
         var_Dump($sSQL);
         $oDB->query($sSQL);

-        echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
+        echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";

 
         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
         $oDB->query($sSQL);
 
         $fDuration = time() - $fStartTime;
 
         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
         $oDB->query($sSQL);
 
         $fDuration = time() - $fStartTime;

-        echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
+        echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";

         if (!$aResult['import-osmosis-all']) exit(0);
 
         if (CONST_Replication_Update_Interval > 60) {
         if (!$aResult['import-osmosis-all']) exit(0);
 
         if (CONST_Replication_Update_Interval > 60) {

-            $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
+            $iSleep = max(0, (strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));

         } else {
         } else {

-            $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
+            $iSleep = max(0, CONST_Replication_Update_Interval-$fDuration);

         }
         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
         sleep($iSleep);
     }
 }
 
         }
         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
         sleep($iSleep);
     }
 }
 

+

 function getosmosistimestamp($sOsmosisConfigDirectory)
 {
     $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
     preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
 function getosmosistimestamp($sOsmosisConfigDirectory)
 {
     $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
     preg_match('#timestamp=(.+)#', $sStateFile, $aResult);

-    return str_replace('\:',':',$aResult[1]);
+    return str_replace('\:', ':', $aResult[1]);

 }
 }