utils/update.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
   5         require_once(CONST_BasePath.'/lib/init-cmd.php');
   6         ini_set('memory_limit', '800M');
   7
   8         $aCMDOptions = array(
   9                 "Import / update / index osm data",
  10                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  11                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  12                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  13
  14                 array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
  15                 array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
  16                 array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
  17                 array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
  18
  19                 array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
  20
  21                 array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
  22                 array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
  23                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  24
  25                 array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
  26                 array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
  27                 array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
  28                 array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
  29
  30                 array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
  31                 array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
  32                 array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
  33
  34                 array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
  35         );
  36         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
  37
  38         if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
  39         if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
  40
  41         date_default_timezone_set('Etc/UTC');
  42
  43         $oDB =& getDB();
  44
  45         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  46         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  47
  48         // cache memory to be used by osm2pgsql, should not be more than the available memory
  49         $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
  50         if ($iCacheMemory + 500 > getTotalMemoryMB())
  51         {
  52                 $iCacheMemory = getCacheMemoryMB();
  53                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  54         }
  55         $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
  56         if (!is_null(CONST_Osm2pgsql_Flatnode_File))
  57         {
  58                 $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
  59         }
  60
  61
  62         if (isset($aResult['import-diff']))
  63         {
  64                 // import diff directly (e.g. from osmosis --rri)
  65                 $sNextFile = $aResult['import-diff'];
  66                 if (!file_exists($sNextFile))
  67                 {
  68                         fail("Cannot open $sNextFile\n");
  69                 }
  70
  71                 // Import the file
  72                 $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
  73                 echo $sCMD."\n";
  74                 exec($sCMD, $sJunk, $iErrorLevel);
  75
  76                 if ($iErrorLevel)
  77                 {
  78                         fail("Error from osm2pgsql, $iErrorLevel\n");
  79                 }
  80
  81                 // Don't update the import status - we don't know what this file contains
  82         }
  83
  84         $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
  85         $bHaveDiff = false;
  86         if (isset($aResult['import-file']) && $aResult['import-file'])
  87         {
  88                 $bHaveDiff = true;
  89                 $sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
  90                 echo $sCMD."\n";
  91                 exec($sCMD, $sJunk, $iErrorLevel);
  92                 if ($iErrorLevel)
  93                 {
  94                         fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
  95                 }
  96         }
  97
  98         $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
  99         $sContentURL = '';
 100         if (isset($aResult['import-node']) && $aResult['import-node'])
 101         {
 102                 if ($bUseOSMApi)
 103                 {
 104                         $sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
 105                 }
 106                 else
 107                 {
 108                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
 109                 }
 110         }
 111         if (isset($aResult['import-way']) && $aResult['import-way'])
 112         {
 113                 if ($bUseOSMApi)
 114                 {
 115                         $sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
 116                 }
 117                 else
 118                 {
 119                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
 120                 }
 121         }
 122         if (isset($aResult['import-relation']) && $aResult['import-relation'])
 123         {
 124                 if ($bUseOSMApi)
 125                 {
 126                         $sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
 127                 }
 128                 else
 129                 {
 130                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
 131                 }
 132         }
 133         if ($sContentURL)
 134         {
 135                 $sModifyXMLstr = file_get_contents($sContentURL);
 136                 $bHaveDiff = true;
 137
 138                 $aSpec = array(
 139                         0 => array("pipe", "r"),  // stdin
 140                         1 => array("pipe", "w"),  // stdout
 141                         2 => array("pipe", "w") // stderr
 142                 );
 143                 $sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
 144                 echo $sCMD."\n";
 145                 $hProc = proc_open($sCMD, $aSpec, $aPipes);
 146                 if (!is_resource($hProc))
 147                 {
 148                         fail("Error converting osm to osc, osmosis failed\n");
 149                 }
 150                 fwrite($aPipes[0], $sModifyXMLstr);
 151                 fclose($aPipes[0]);
 152                 $sOut = stream_get_contents($aPipes[1]);
 153                 if ($aResult['verbose']) echo $sOut;
 154                 fclose($aPipes[1]);
 155                 $sErrors = stream_get_contents($aPipes[2]);
 156                 if ($aResult['verbose']) echo $sErrors;
 157                 fclose($aPipes[2]);
 158                 if ($iError = proc_close($hProc))
 159                 {
 160                         echo $sOut;
 161                         echo $sErrors;
 162                         fail("Error converting osm to osc, osmosis returned: $iError\n");
 163                 }
 164         }
 165
 166         if ($bHaveDiff)
 167         {
 168                 // import generated change file
 169                 $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
 170                 echo $sCMD."\n";
 171                 exec($sCMD, $sJunk, $iErrorLevel);
 172                 if ($iErrorLevel)
 173                 {
 174                         fail("osm2pgsql exited with error level $iErrorLevel\n");
 175                 }
 176         }
 177
 178         if ($aResult['deduplicate'])
 179         {
 180
 181                 $pgver = (float) CONST_Postgresql_Version;
 182                 if ($pgver < 9.3) {
 183                         fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
 184                 }
 185
 186                 $oDB =& getDB();
 187                 $sSQL = 'select partition from country_name order by country_code';
 188                 $aPartitions = $oDB->getCol($sSQL);
 189                 if (PEAR::isError($aPartitions))
 190                 {
 191                         fail($aPartitions->getMessage());
 192                 }
 193                 $aPartitions[] = 0;
 194
 195                 $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
 196                 $aDuplicateTokens = $oDB->getAll($sSQL);
 197                 foreach($aDuplicateTokens as $aToken)
 198                 {
 199                         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
 200                         echo "Deduping ".$aToken['word_token']."\n";
 201                         $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
 202                         $aTokenSet = $oDB->getAll($sSQL);
 203                         if (PEAR::isError($aTokenSet))
 204                         {
 205                                 var_dump($aTokenSet, $sSQL);
 206                                 exit(1);
 207                         }
 208
 209                         $aKeep = array_shift($aTokenSet);
 210                         $iKeepID = $aKeep['word_id'];
 211
 212                         foreach($aTokenSet as $aRemove)
 213                         {
 214                                 $sSQL = "update search_name set";
 215                                 $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
 216                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
 217                                 $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
 218                                 $x = $oDB->query($sSQL);
 219                                 if (PEAR::isError($x))
 220                                 {
 221                                         var_dump($x);
 222                                         exit(1);
 223                                 }
 224
 225                                 $sSQL = "update search_name set";
 226                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
 227                                 $sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
 228                                 $x = $oDB->query($sSQL);
 229                                 if (PEAR::isError($x))
 230                                 {
 231                                         var_dump($x);
 232                                         exit(1);
 233                                 }
 234
 235                                 $sSQL = "update location_area_country set";
 236                                 $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
 237                                 $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
 238                                 $x = $oDB->query($sSQL);
 239                                 if (PEAR::isError($x))
 240                                 {
 241                                         var_dump($x);
 242                                         exit(1);
 243                                 }
 244
 245                                 foreach ($aPartitions as $sPartition)
 246                                 {
 247                                         $sSQL = "update search_name_".$sPartition." set";
 248                                         $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
 249                                         $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
 250                                         $x = $oDB->query($sSQL);
 251                                         if (PEAR::isError($x))
 252                                         {
 253                                                 var_dump($x);
 254                                                 exit(1);
 255                                         }
 256
 257                                         $sSQL = "update location_area_country set";
 258                                         $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
 259                                         $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
 260                                         $x = $oDB->query($sSQL);
 261                                         if (PEAR::isError($x))
 262                                         {
 263                                                 var_dump($x);
 264                                                 exit(1);
 265                                         }
 266                                 }
 267
 268                                 $sSQL = "delete from word where word_id = ".$aRemove['word_id'];
 269                                 $x = $oDB->query($sSQL);
 270                                 if (PEAR::isError($x))
 271                                 {
 272                                         var_dump($x);
 273                                         exit(1);
 274                                 }
 275                         }
 276                 }
 277         }
 278
 279         if ($aResult['index'])
 280         {
 281                 passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 282         }
 283
 284         if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
 285         {
 286
 287                 if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
 288                         fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
 289                 }
 290
 291                 $sImportFile = CONST_BasePath.'/data/osmosischange.osc';
 292                 $sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
 293                 $sCMDDownload = CONST_Osmosis_Binary.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
 294                 $sCMDCheckReplicationLag = CONST_Osmosis_Binary.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
 295                 $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
 296                 $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 297
 298                 while(true)
 299                 {
 300                         $fStartTime = time();
 301                         $iFileSize = 1001;
 302
 303                         if (!file_exists($sImportFile))
 304                         {
 305                                 // First check if there are new updates published (except for minutelies - there's always new diffs to process)
 306                                 if ( CONST_Replication_Update_Interval > 60 )
 307                                 {
 308
 309                                         unset($aReplicationLag);
 310                                         exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
 311                                         while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
 312                                         {
 313                                                 if ($iErrorLevel)
 314                                                 {
 315                                                         echo "Error: $iErrorLevel. ";
 316                                                         echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
 317                                                 }
 318                                                 else
 319                                                 {
 320                                                         echo ".";
 321                                                 }
 322                                                 sleep(CONST_Replication_Recheck_Interval);
 323                                                 unset($aReplicationLag);
 324                                                 exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
 325                                         }
 326                                         // There are new replication files - use osmosis to download the file
 327                                         echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
 328                                 }
 329                                 $fStartTime = time();
 330                                 $fCMDStartTime = time();
 331                                 echo $sCMDDownload."\n";
 332                                 exec($sCMDDownload, $sJunk, $iErrorLevel);
 333                                 while ($iErrorLevel > 0)
 334                                 {
 335                                         echo "Error: $iErrorLevel\n";
 336                                         sleep(60);
 337                                         echo 'Re-trying: '.$sCMDDownload."\n";
 338                                         exec($sCMDDownload, $sJunk, $iErrorLevel);
 339                                 }
 340                                 $iFileSize = filesize($sImportFile);
 341                                 $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 342                                 $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
 343                                 var_Dump($sSQL);
 344                                 $oDB->query($sSQL);
 345                                 echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 346                         }
 347
 348                         $iFileSize = filesize($sImportFile);
 349                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 350
 351                         // Import the file
 352                         $fCMDStartTime = time();
 353                         echo $sCMDImport."\n";
 354                         exec($sCMDImport, $sJunk, $iErrorLevel);
 355                         if ($iErrorLevel)
 356                         {
 357                                 echo "Error: $iErrorLevel\n";
 358                                 exit($iErrorLevel);
 359                         }
 360                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
 361                         var_Dump($sSQL);
 362                         $oDB->query($sSQL);
 363                         echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 364
 365                         // Archive for debug?
 366                         unlink($sImportFile);
 367
 368                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 369
 370                         // Index file
 371                         $sThisIndexCmd = $sCMDIndex;
 372                         $fCMDStartTime = time();
 373
 374                         if (!$aResult['no-index'])
 375                         {
 376                                 echo "$sThisIndexCmd\n";
 377                                 exec($sThisIndexCmd, $sJunk, $iErrorLevel);
 378                                 if ($iErrorLevel)
 379                                 {
 380                                         echo "Error: $iErrorLevel\n";
 381                                         exit($iErrorLevel);
 382                                 }
 383                         }
 384
 385                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
 386                         var_Dump($sSQL);
 387                         $oDB->query($sSQL);
 388                         echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 389
 390                         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
 391                         $oDB->query($sSQL);
 392
 393                         $fDuration = time() - $fStartTime;
 394                         echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
 395                         if (!$aResult['import-osmosis-all']) exit(0);
 396
 397                         if ( CONST_Replication_Update_Interval > 60 )
 398                         {
 399                                 $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
 400                         }
 401                         else
 402                         {
 403                                 $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
 404                         }
 405                         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
 406                         sleep($iSleep);
 407                 }
 408         }
 409
 410         function getosmosistimestamp($sOsmosisConfigDirectory)
 411         {
 412                 $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
 413                 preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
 414                 return str_replace('\:',':',$aResult[1]);
 415         }