X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/f05ea577f42c5ad4a62ff25ec62fbb60c550ee26..6c12fc971e0e6141572ad50adf79533b4fc87d1a:/utils/update.php?ds=sidebyside

diff --git a/utils/update.php b/utils/update.php
index 783f55d4..8137584c 100755
--- a/utils/update.php
+++ b/utils/update.php
@@ -5,34 +5,35 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
 require_once(CONST_BasePath.'/lib/init-cmd.php');
 ini_set('memory_limit', '800M');
 
-$aCMDOptions = array(
-    "Import / update / index osm data",
-    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
-    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
-    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
-
-    array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
-    array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
-    array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
-    array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
-
-    array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
-
-    array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
-    array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
-    array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
-
-    array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
-    array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
-    array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
-    array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
-
-    array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
-    array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
-    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
-
-    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
-);
+$aCMDOptions
+= array(
+   "Import / update / index osm data",
+   array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+   array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+   array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+
+   array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
+   array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
+   array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
+   array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
+
+   array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
+
+   array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
+   array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
+   array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
+
+   array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
+   array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
+   array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
+   array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
+
+   array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
+   array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
+   array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
+
+   array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
+  );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
 if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
@@ -119,10 +120,10 @@ if ($sContentURL) {
     $bHaveDiff = true;
 
     $aSpec = array(
-        0 => array("pipe", "r"),  // stdin
-        1 => array("pipe", "w"),  // stdout
-        2 => array("pipe", "w") // stderr
-    );
+              0 => array("pipe", "r"),  // stdin
+              1 => array("pipe", "w"),  // stdout
+              2 => array("pipe", "w") // stderr
+             );
     $sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
     echo $sCMD."\n";
     $hProc = proc_open($sCMD, $aSpec, $aPipes);
@@ -165,12 +166,17 @@ if ($aResult['deduplicate']) {
     $aPartitions = chksql($oDB->getCol($sSQL));
     $aPartitions[] = 0;
 
-    $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
+    $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '";
+    $sSQL .= " and class is null and type is null and country_code is null";
+    $sSQL .= " group by word_token having count(*) > 1 order by word_token";
     $aDuplicateTokens = chksql($oDB->getAll($sSQL));
     foreach ($aDuplicateTokens as $aToken) {
         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
         echo "Deduping ".$aToken['word_token']."\n";
-        $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
+        $sSQL = "select word_id,";
+        $sSQL .= " (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num";
+        $sSQL .= " from word where word_token = '".$aToken['word_token'];
+        $sSQL .= "' and class is null and type is null and country_code is null order by num desc";
         $aTokenSet = chksql($oDB->getAll($sSQL));
 
         $aKeep = array_shift($aTokenSet);
@@ -236,7 +242,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             // First check if there are new updates published (except for minutelies - there's always new diffs to process)
             if (CONST_Replication_Update_Interval > 60) {
                 unset($aReplicationLag);
-                exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+                exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
                 while ($iErrorLevel > 0 || $aReplicationLag[0] < 1) {
                     if ($iErrorLevel) {
                         echo "Error: $iErrorLevel. ";
@@ -246,7 +252,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                     }
                     sleep(CONST_Replication_Recheck_Interval);
                     unset($aReplicationLag);
-                    exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+                    exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
                 }
                 // There are new replication files - use osmosis to download the file
                 echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
@@ -263,10 +269,10 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             }
             $iFileSize = filesize($sImportFile);
             $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
-            $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
+            $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
             var_Dump($sSQL);
             $oDB->query($sSQL);
-            echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
+            echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
         }
 
         $iFileSize = filesize($sImportFile);
@@ -280,10 +286,10 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             echo "Error: $iErrorLevel\n";
             exit($iErrorLevel);
         }
-        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
+        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
         var_Dump($sSQL);
         $oDB->query($sSQL);
-        echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
+        echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
 
         // Archive for debug?
         unlink($sImportFile);
@@ -292,6 +298,15 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
         // Index file
         $sThisIndexCmd = $sCMDIndex;
+        if (!isset($aResult['index-instances'])) {
+            if (getLoadAverage() < 24)
+                $iIndexInstances = 2;
+            else
+                $iIndexInstances = 1;
+        } else
+            $iIndexInstances = $aResult['index-instances'];
+
+        $sThisIndexCmd = $sCMDIndex.' -t '.$iIndexInstances;
         $fCMDStartTime = time();
 
         if (!$aResult['no-index']) {
@@ -303,31 +318,32 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             }
         }
 
-        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+        $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
         var_Dump($sSQL);
         $oDB->query($sSQL);
-        echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
+        echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
 
         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
         $oDB->query($sSQL);
 
         $fDuration = time() - $fStartTime;
-        echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
+        echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";
         if (!$aResult['import-osmosis-all']) exit(0);
 
         if (CONST_Replication_Update_Interval > 60) {
-            $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
+            $iSleep = max(0, (strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
         } else {
-            $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
+            $iSleep = max(0, CONST_Replication_Update_Interval-$fDuration);
         }
         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
         sleep($iSleep);
     }
 }
 
+
 function getosmosistimestamp($sOsmosisConfigDirectory)
 {
     $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
     preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
-    return str_replace('\:',':',$aResult[1]);
+    return str_replace('\:', ':', $aResult[1]);
 }