utils/update.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
   5         require_once(CONST_BasePath.'/lib/init-cmd.php');
   6         ini_set('memory_limit', '800M');
   7
   8         $aCMDOptions = array(
   9                 "Import / update / index osm data",
  10                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  11                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  12                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  13
  14                 array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
  15                 array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
  16                 array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
  17                 array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
  18
  19                 array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
  20
  21                 array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
  22                 array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
  23                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  24
  25                 array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
  26                 array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
  27                 array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
  28                 array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
  29
  30                 array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
  31                 array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
  32                 array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
  33
  34                 array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
  35         );
  36         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
  37
  38         if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
  39         if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
  40
  41         date_default_timezone_set('Etc/UTC');
  42
  43         $oDB =& getDB();
  44
  45         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  46         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  47
  48         // cache memory to be used by osm2pgsql, should not be more than the available memory
  49         $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
  50         if ($iCacheMemory + 500 > getTotalMemoryMB())
  51         {
  52                 $iCacheMemory = getCacheMemoryMB();
  53                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  54         }
  55         $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
  56         if (!is_null(CONST_Osm2pgsql_Flatnode_File))
  57         {
  58                 $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
  59         }
  60
  61
  62         if (isset($aResult['import-diff']))
  63         {
  64                 // import diff directly (e.g. from osmosis --rri)
  65                 $sNextFile = $aResult['import-diff'];
  66                 if (!file_exists($sNextFile))
  67                 {
  68                         fail("Cannot open $sNextFile\n");
  69                 }
  70
  71                 // Import the file
  72                 $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
  73                 echo $sCMD."\n";
  74                 exec($sCMD, $sJunk, $iErrorLevel);
  75
  76                 if ($iErrorLevel)
  77                 {
  78                         fail("Error from osm2pgsql, $iErrorLevel\n");
  79                 }
  80
  81                 // Don't update the import status - we don't know what this file contains
  82         }
  83
  84         $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
  85         $bHaveDiff = false;
  86         if (isset($aResult['import-file']) && $aResult['import-file'])
  87         {
  88                 $bHaveDiff = true;
  89                 $sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
  90                 echo $sCMD."\n";
  91                 exec($sCMD, $sJunk, $iErrorLevel);
  92                 if ($iErrorLevel)
  93                 {
  94                         fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
  95                 }
  96         }
  97
  98         $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
  99         $sContentURL = '';
 100         if (isset($aResult['import-node']) && $aResult['import-node'])
 101         {
 102                 if ($bUseOSMApi)
 103                 {
 104                         $sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
 105                 }
 106                 else
 107                 {
 108                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
 109                 }
 110         }
 111         if (isset($aResult['import-way']) && $aResult['import-way'])
 112         {
 113                 if ($bUseOSMApi)
 114                 {
 115                         $sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
 116                 }
 117                 else
 118                 {
 119                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
 120                 }
 121         }
 122         if (isset($aResult['import-relation']) && $aResult['import-relation'])
 123         {
 124                 if ($bUseOSMApi)
 125                 {
 126                         $sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
 127                 }
 128                 else
 129                 {
 130                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
 131                 }
 132         }
 133         if ($sContentURL)
 134         {
 135                 $sModifyXMLstr = file_get_contents($sContentURL);
 136                 $bHaveDiff = true;
 137
 138                 $aSpec = array(
 139                         0 => array("pipe", "r"),  // stdin
 140                         1 => array("pipe", "w"),  // stdout
 141                         2 => array("pipe", "w") // stderr
 142                 );
 143                 $sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
 144                 echo $sCMD."\n";
 145                 $hProc = proc_open($sCMD, $aSpec, $aPipes);
 146                 if (!is_resource($hProc))
 147                 {
 148                         fail("Error converting osm to osc, osmosis failed\n");
 149                 }
 150                 fwrite($aPipes[0], $sModifyXMLstr);
 151                 fclose($aPipes[0]);
 152                 $sOut = stream_get_contents($aPipes[1]);
 153                 if ($aResult['verbose']) echo $sOut;
 154                 fclose($aPipes[1]);
 155                 $sErrors = stream_get_contents($aPipes[2]);
 156                 if ($aResult['verbose']) echo $sErrors;
 157                 fclose($aPipes[2]);
 158                 if ($iError = proc_close($hProc))
 159                 {
 160                         echo $sOut;
 161                         echo $sErrors;
 162                         fail("Error converting osm to osc, osmosis returned: $iError\n");
 163                 }
 164         }
 165
 166         if ($bHaveDiff)
 167         {
 168                 // import generated change file
 169                 $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
 170                 echo $sCMD."\n";
 171                 exec($sCMD, $sJunk, $iErrorLevel);
 172                 if ($iErrorLevel)
 173                 {
 174                         fail("osm2pgsql exited with error level $iErrorLevel\n");
 175                 }
 176         }
 177
 178         if ($aResult['deduplicate'])
 179         {
 180
 181                 if (getPostgresVersion() < 9.3)
 182                 {
 183                         fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
 184                 }
 185
 186                 $oDB =& getDB();
 187                 $sSQL = 'select partition from country_name order by country_code';
 188                 $aPartitions = chksql($oDB->getCol($sSQL));
 189                 $aPartitions[] = 0;
 190
 191                 $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
 192                 $aDuplicateTokens = chksql($oDB->getAll($sSQL));
 193                 foreach($aDuplicateTokens as $aToken)
 194                 {
 195                         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
 196                         echo "Deduping ".$aToken['word_token']."\n";
 197                         $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
 198                         $aTokenSet = chksql($oDB->getAll($sSQL));
 199
 200                         $aKeep = array_shift($aTokenSet);
 201                         $iKeepID = $aKeep['word_id'];
 202
 203                         foreach($aTokenSet as $aRemove)
 204                         {
 205                                 $sSQL = "update search_name set";
 206                                 $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
 207                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
 208                                 $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
 209                                 chksql($oDB->query($sSQL));
 210
 211                                 $sSQL = "update search_name set";
 212                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
 213                                 $sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
 214                                 chksql($oDB->query($sSQL));
 215
 216                                 $sSQL = "update location_area_country set";
 217                                 $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
 218                                 $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
 219                                 chksql($oDB->query($sSQL));
 220
 221                                 foreach ($aPartitions as $sPartition)
 222                                 {
 223                                         $sSQL = "update search_name_".$sPartition." set";
 224                                         $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
 225                                         $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
 226                                         chksql($oDB->query($sSQL));
 227
 228                                         $sSQL = "update location_area_country set";
 229                                         $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
 230                                         $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
 231                                         chksql($oDB->query($sSQL));
 232                                 }
 233
 234                                 $sSQL = "delete from word where word_id = ".$aRemove['word_id'];
 235                                 chksql($oDB->query($sSQL));
 236                         }
 237                 }
 238         }
 239
 240         if ($aResult['index'])
 241         {
 242                 passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 243         }
 244
 245         if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
 246         {
 247
 248                 if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
 249                         fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
 250                 }
 251
 252                 $sImportFile = CONST_BasePath.'/data/osmosischange.osc';
 253                 $sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
 254                 $sCMDDownload = CONST_Osmosis_Binary.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
 255                 $sCMDCheckReplicationLag = CONST_Osmosis_Binary.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
 256                 $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
 257                 $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 258
 259                 while(true)
 260                 {
 261                         $fStartTime = time();
 262                         $iFileSize = 1001;
 263
 264                         if (!file_exists($sImportFile))
 265                         {
 266                                 // First check if there are new updates published (except for minutelies - there's always new diffs to process)
 267                                 if ( CONST_Replication_Update_Interval > 60 )
 268                                 {
 269
 270                                         unset($aReplicationLag);
 271                                         exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
 272                                         while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
 273                                         {
 274                                                 if ($iErrorLevel)
 275                                                 {
 276                                                         echo "Error: $iErrorLevel. ";
 277                                                         echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
 278                                                 }
 279                                                 else
 280                                                 {
 281                                                         echo ".";
 282                                                 }
 283                                                 sleep(CONST_Replication_Recheck_Interval);
 284                                                 unset($aReplicationLag);
 285                                                 exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
 286                                         }
 287                                         // There are new replication files - use osmosis to download the file
 288                                         echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
 289                                 }
 290                                 $fStartTime = time();
 291                                 $fCMDStartTime = time();
 292                                 echo $sCMDDownload."\n";
 293                                 exec($sCMDDownload, $sJunk, $iErrorLevel);
 294                                 while ($iErrorLevel > 0)
 295                                 {
 296                                         echo "Error: $iErrorLevel\n";
 297                                         sleep(60);
 298                                         echo 'Re-trying: '.$sCMDDownload."\n";
 299                                         exec($sCMDDownload, $sJunk, $iErrorLevel);
 300                                 }
 301                                 $iFileSize = filesize($sImportFile);
 302                                 $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 303                                 $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
 304                                 var_Dump($sSQL);
 305                                 $oDB->query($sSQL);
 306                                 echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 307                         }
 308
 309                         $iFileSize = filesize($sImportFile);
 310                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 311
 312                         // Import the file
 313                         $fCMDStartTime = time();
 314                         echo $sCMDImport."\n";
 315                         exec($sCMDImport, $sJunk, $iErrorLevel);
 316                         if ($iErrorLevel)
 317                         {
 318                                 echo "Error: $iErrorLevel\n";
 319                                 exit($iErrorLevel);
 320                         }
 321                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
 322                         var_Dump($sSQL);
 323                         $oDB->query($sSQL);
 324                         echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 325
 326                         // Archive for debug?
 327                         unlink($sImportFile);
 328
 329                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 330
 331                         // Index file
 332                         if (!isset($aResult['index-instances']))
 333                         {
 334                                 if (getLoadAverage() < 24)
 335                                         $iIndexInstances = 2;
 336                                 else
 337                                         $iIndexInstances = 1;
 338                         } else
 339                                 $iIndexInstances = $aResult['index-instances'];
 340
 341                         $sThisIndexCmd = $sCMDIndex.' -t '.$iIndexInstances;
 342                         $fCMDStartTime = time();
 343
 344                         if (!$aResult['no-index'])
 345                         {
 346                                 echo "$sThisIndexCmd\n";
 347                                 exec($sThisIndexCmd, $sJunk, $iErrorLevel);
 348                                 if ($iErrorLevel)
 349                                 {
 350                                         echo "Error: $iErrorLevel\n";
 351                                         exit($iErrorLevel);
 352                                 }
 353                         }
 354
 355                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
 356                         var_Dump($sSQL);
 357                         $oDB->query($sSQL);
 358                         echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 359
 360                         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
 361                         $oDB->query($sSQL);
 362
 363                         $fDuration = time() - $fStartTime;
 364                         echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
 365                         if (!$aResult['import-osmosis-all']) exit(0);
 366
 367                         if ( CONST_Replication_Update_Interval > 60 )
 368                         {
 369                                 $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
 370                         }
 371                         else
 372                         {
 373                                 $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
 374                         }
 375                         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
 376                         sleep($iSleep);
 377                 }
 378         }
 379
 380         function getosmosistimestamp($sOsmosisConfigDirectory)
 381         {
 382                 $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
 383                 preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
 384                 return str_replace('\:',':',$aResult[1]);
 385         }