utils/update.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
   5         require_once(CONST_BasePath.'/lib/init-cmd.php');
   6         ini_set('memory_limit', '800M');
   7
   8         $aCMDOptions = array(
   9                 "Import / update / index osm data",
  10                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  11                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  12                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  13
  14                 array('max-load', '', 0, 1, 1, 1, 'float', 'Maximum load average - indexing is paused if this is exceeded'),
  15                 array('max-blocking', '', 0, 1, 1, 1, 'int', 'Maximum blocking processes - indexing is aborted / paused if this is exceeded'),
  16
  17                 array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
  18                 array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
  19                 array('no-npi', '', 0, 1, 0, 0, 'bool', 'Do not write npi index files'),
  20                 array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
  21
  22                 array('import-npi-all', '', 0, 1, 0, 0, 'bool', 'Import npi pre-indexed files'),
  23
  24                 array('import-hourly', '', 0, 1, 0, 0, 'bool', 'Import hourly diffs'),
  25                 array('import-daily', '', 0, 1, 0, 0, 'bool', 'Import daily diffs'),
  26                 array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
  27
  28                 array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
  29                 array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
  30                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  31
  32                 array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
  33                 array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
  34                 array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
  35                 array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
  36
  37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
  38                 array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
  39                 array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
  40                 array('index-estrate', '', 0, 1, 1, 1, 'int', 'Estimated indexed items per second (def:30)'),
  41
  42                 array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
  43         );
  44         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
  45
  46         if ($aResult['import-hourly'] + $aResult['import-daily'] + isset($aResult['import-diff']) > 1)
  47         {
  48                 showUsage($aCMDOptions, true, 'Select either import of hourly or daily');
  49         }
  50
  51         if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
  52         if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
  53
  54 /*
  55         // Lock to prevent multiple copies running
  56         if (exec('/bin/ps uww | grep '.basename(__FILE__).' | grep -v /dev/null | grep -v grep -c', $aOutput2, $iResult) > 1)
  57         {
  58                 fail("Copy already running\n");
  59         }
  60         if (!isset($aResult['max-load'])) $aResult['max-load'] = 1.9;
  61         if (!isset($aResult['max-blocking'])) $aResult['max-blocking'] = 3;
  62         if (getBlockingProcesses() > $aResult['max-blocking'])
  63         {
  64                 fail("Too many blocking processes for import\n");
  65         }
  66 */
  67
  68         date_default_timezone_set('Etc/UTC');
  69
  70         $oDB =& getDB();
  71
  72         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  73         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  74
  75         // cache memory to be used by osm2pgsql, should not be more than the available memory
  76         $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
  77         if ($iCacheMemory + 500 > getTotalMemoryMB())
  78         {
  79                 $iCacheMemory = getCacheMemoryMB();
  80                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  81         }
  82         $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
  83         if (!is_null(CONST_Osm2pgsql_Flatnode_File))
  84         {
  85                 $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
  86         }
  87
  88
  89         $bFirst = true;
  90         $bContinue = $aResult['import-all'];
  91         while ($bContinue || $bFirst)
  92         {
  93                 $bFirst = false;
  94
  95                 if ($aResult['import-hourly'])
  96                 {
  97                         // Mirror the hourly diffs
  98                         exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/hourly');
  99                         $sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDDHH24\')||\'-\'||TO_CHAR(lastimportdate+\'1 hour\'::interval,\'YYYYMMDDHH24\')||\'.osc.gz\' from import_status');
 100                         $sNextFile = $sMirrorDir.'planet.openstreetmap.org/hourly/'.$sNextFile;
 101                         $sUpdateSQL = 'update import_status set lastimportdate = lastimportdate+\'1 hour\'::interval';
 102                 }
 103
 104                 if ($aResult['import-daily'])
 105                 {
 106                         // Mirror the daily diffs
 107                         exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/daily');
 108                         $sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDD\')||\'-\'||TO_CHAR(lastimportdate+\'1 day\'::interval,\'YYYYMMDD\')||\'.osc.gz\' from import_status');
 109                         $sNextFile = $sMirrorDir.'planet.openstreetmap.org/daily/'.$sNextFile;
 110                         $sUpdateSQL = 'update import_status set lastimportdate = lastimportdate::date + 1';
 111                 }
 112
 113                 if (isset($aResult['import-diff']))
 114                 {
 115                         // import diff directly (e.g. from osmosis --rri)
 116                         $sNextFile = $aResult['import-diff'];
 117                         if (!file_exists($sNextFile))
 118                         {
 119                                 fail("Cannot open $sNextFile\n");
 120                         }
 121                         // Don't update the import status - we don't know what this file contains
 122                         $sUpdateSQL = 'update import_status set lastimportdate = now() where false';
 123                 }
 124
 125                 // Missing file is not an error - it might not be created yet
 126                 if (($aResult['import-hourly'] || $aResult['import-daily'] || isset($aResult['import-diff'])) && file_exists($sNextFile))
 127                 {
 128                         // Import the file
 129                         $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
 130                         echo $sCMD."\n";
 131                         exec($sCMD, $sJunk, $iErrorLevel);
 132
 133                         if ($iErrorLevel)
 134                         {
 135                                 fail("Error from osm2pgsql, $iErrorLevel\n");
 136                         }
 137
 138                         // Move the date onwards
 139                         $oDB->query($sUpdateSQL);
 140                 }
 141                 else
 142                 {
 143                         $bContinue = false;
 144                 }
 145         }
 146
 147         $bModifyXML = false;
 148         $sModifyXMLstr = '';
 149         $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
 150         if (isset($aResult['import-file']) && $aResult['import-file'])
 151         {
 152                 $bModifyXML = true;
 153         }
 154         if (isset($aResult['import-node']) && $aResult['import-node'])
 155         {
 156                 $bModifyXML = true;
 157                 if ($bUseOSMApi)
 158                 {
 159                         $sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']);
 160                 }
 161                 else
 162                 {
 163                         $sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;');
 164                 }
 165         }
 166         if (isset($aResult['import-way']) && $aResult['import-way'])
 167         {
 168                 $bModifyXML = true;
 169                 if ($bUseOSMApi)
 170                 {
 171                         $sCmd = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
 172                 }
 173                 else
 174                 {
 175                         $sCmd = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
 176                 }
 177                 $sModifyXMLstr = file_get_contents($sCmd);
 178         }
 179         if (isset($aResult['import-relation']) && $aResult['import-relation'])
 180         {
 181                 $bModifyXML = true;
 182                 if ($bUseOSMApi)
 183                 {
 184                         $sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full');
 185                 }
 186                 else
 187                 {
 188                         $sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;');
 189                 }
 190         }
 191         if ($bModifyXML)
 192         {
 193                 // derive change from normal osm file with osmosis
 194                 $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
 195                 if (isset($aResult['import-file']) && $aResult['import-file'])
 196                 {
 197                         $sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
 198                         echo $sCMD."\n";
 199                         exec($sCMD, $sJunk, $iErrorLevel);
 200                         if ($iErrorLevel)
 201                         {
 202                                 fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
 203                         }
 204                 }
 205                 else
 206                 {
 207                         $aSpec = array(
 208                                 0 => array("pipe", "r"),  // stdin
 209                                 1 => array("pipe", "w"),  // stdout
 210                                 2 => array("pipe", "w") // stderr
 211                         );
 212                         $sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
 213                         echo $sCMD."\n";
 214                         $hProc = proc_open($sCMD, $aSpec, $aPipes);
 215                         if (!is_resource($hProc))
 216                         {
 217                                 fail("Error converting osm to osc, osmosis failed\n");
 218                         }
 219                         fwrite($aPipes[0], $sModifyXMLstr);
 220                         fclose($aPipes[0]);
 221                         $sOut = stream_get_contents($aPipes[1]);
 222                         if ($aResult['verbose']) echo $sOut;
 223                         fclose($aPipes[1]);
 224                         $sErrors = stream_get_contents($aPipes[2]);
 225                         if ($aResult['verbose']) echo $sErrors;
 226                         fclose($aPipes[2]);
 227                         if ($iError = proc_close($hProc))
 228                         {
 229                                 echo "Error converting osm to osc, osmosis returned: $iError\n";
 230                                 echo $sOut;
 231                                 echo $sErrors;
 232                                 exit(-1);
 233                         }
 234                 }
 235
 236                 // import generated change file
 237                 $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
 238                 echo $sCMD."\n";
 239                 exec($sCMD, $sJunk, $iErrorLevel);
 240                 if ($iErrorLevel)
 241                 {
 242                         fail("osm2pgsql exited with error level $iErrorLevel\n");
 243                 }
 244         }
 245
 246         if ($aResult['deduplicate'])
 247         {
 248
 249                 $pgver = (float) CONST_Postgresql_Version;
 250                 if ($pgver < 9.3) {
 251                         fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
 252                 }
 253
 254                 $oDB =& getDB();
 255                 $sSQL = 'select partition from country_name order by country_code';
 256                 $aPartitions = $oDB->getCol($sSQL);
 257                 if (PEAR::isError($aPartitions))
 258                 {
 259                         fail($aPartitions->getMessage());
 260                 }
 261                 $aPartitions[] = 0;
 262
 263                 $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
 264                 $aDuplicateTokens = $oDB->getAll($sSQL);
 265                 foreach($aDuplicateTokens as $aToken)
 266                 {
 267                         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
 268                         echo "Deduping ".$aToken['word_token']."\n";
 269                         $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
 270                         $aTokenSet = $oDB->getAll($sSQL);
 271                         if (PEAR::isError($aTokenSet))
 272                         {
 273                                 var_dump($aTokenSet, $sSQL);
 274                                 exit(1);
 275                         }
 276
 277                         $aKeep = array_shift($aTokenSet);
 278                         $iKeepID = $aKeep['word_id'];
 279
 280                         foreach($aTokenSet as $aRemove)
 281                         {
 282                                 $sSQL = "update search_name set";
 283                                 $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
 284                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
 285                                 $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
 286                                 $x = $oDB->query($sSQL);
 287                                 if (PEAR::isError($x))
 288                                 {
 289                                         var_dump($x);
 290                                         exit(1);
 291                                 }
 292
 293                                 $sSQL = "update search_name set";
 294                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
 295                                 $sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
 296                                 $x = $oDB->query($sSQL);
 297                                 if (PEAR::isError($x))
 298                                 {
 299                                         var_dump($x);
 300                                         exit(1);
 301                                 }
 302
 303                                 $sSQL = "update location_area_country set";
 304                                 $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
 305                                 $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
 306                                 $x = $oDB->query($sSQL);
 307                                 if (PEAR::isError($x))
 308                                 {
 309                                         var_dump($x);
 310                                         exit(1);
 311                                 }
 312
 313                                 foreach ($aPartitions as $sPartition)
 314                                 {
 315                                         $sSQL = "update search_name_".$sPartition." set";
 316                                         $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
 317                                         $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
 318                                         $x = $oDB->query($sSQL);
 319                                         if (PEAR::isError($x))
 320                                         {
 321                                                 var_dump($x);
 322                                                 exit(1);
 323                                         }
 324
 325                                         $sSQL = "update location_area_country set";
 326                                         $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
 327                                         $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
 328                                         $x = $oDB->query($sSQL);
 329                                         if (PEAR::isError($x))
 330                                         {
 331                                                 var_dump($x);
 332                                                 exit(1);
 333                                         }
 334                                 }
 335
 336                                 $sSQL = "delete from word where word_id = ".$aRemove['word_id'];
 337                                 $x = $oDB->query($sSQL);
 338                                 if (PEAR::isError($x))
 339                                 {
 340                                         var_dump($x);
 341                                         exit(1);
 342                                 }
 343                         }
 344
 345                 }
 346         }
 347
 348         if ($aResult['index'])
 349         {
 350                 passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 351         }
 352
 353         if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
 354         {
 355
 356                 if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
 357                         fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
 358                 }
 359
 360                 $sImportFile = CONST_BasePath.'/data/osmosischange.osc';
 361                 $sOsmosisCMD = CONST_Osmosis_Binary;
 362                 $sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
 363                 $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
 364                 $sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
 365                 $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
 366                 $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 367                 if (!$aResult['no-npi']) {
 368                         $sCMDIndex .= '-F ';
 369                 }
 370                 while(true)
 371                 {
 372                         $fStartTime = time();
 373                         $iFileSize = 1001;
 374
 375                         // Logic behind this is that osm2pgsql locks the database quite a bit
 376                         // So it is better to import lots of small files
 377                         // But indexing works most efficiently on large amounts of data
 378                         // So do lots of small imports and a BIG index
 379
 380 //                      while($aResult['import-osmosis-all'] && $iFileSize > 1000)
 381 //                      {
 382                                 if (!file_exists($sImportFile))
 383                                 {
 384                                         // First check if there are new updates published (except for minutelies - there's always new diffs to process)
 385                                         if ( CONST_Replication_Update_Interval > 60 )
 386                                         {
 387
 388                                                 unset($aReplicationLag);
 389                                                 exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
 390                                                 while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
 391                                                 {
 392                                                         if ($iErrorLevel)
 393                                                         {
 394                                                                 echo "Error: $iErrorLevel. ";
 395                                                                 echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
 396                                                         }
 397                                                         else
 398                                                         {
 399                                                                 echo ".";
 400                                                         }
 401                                                         sleep(CONST_Replication_Recheck_Interval);
 402                                                         unset($aReplicationLag);
 403                                                         exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
 404                                                 }
 405                                                 // There are new replication files - use osmosis to download the file
 406                                                 echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
 407                                         }
 408                                         $fStartTime = time();
 409                                         $fCMDStartTime = time();
 410                                         echo $sCMDDownload."\n";
 411                                         exec($sCMDDownload, $sJunk, $iErrorLevel);
 412                                         while ($iErrorLevel > 0)
 413                                         {
 414                                                 echo "Error: $iErrorLevel\n";
 415                                                 sleep(60);
 416                                                 echo 'Re-trying: '.$sCMDDownload."\n";
 417                                                 exec($sCMDDownload, $sJunk, $iErrorLevel);
 418                                         }
 419                                         $iFileSize = filesize($sImportFile);
 420                                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 421                                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
 422                                         var_Dump($sSQL);
 423                                         $oDB->query($sSQL);
 424                                         echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 425                                 }
 426
 427                                 $iFileSize = filesize($sImportFile);
 428                                 $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 429
 430                                 // Import the file
 431                                 $fCMDStartTime = time();
 432                                 echo $sCMDImport."\n";
 433                                 exec($sCMDImport, $sJunk, $iErrorLevel);
 434                                 if ($iErrorLevel)
 435                                 {
 436                                         echo "Error: $iErrorLevel\n";
 437                                         exit($iErrorLevel);
 438                                 }
 439                                 $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
 440                                 var_Dump($sSQL);
 441                                 $oDB->query($sSQL);
 442                                 echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 443
 444                                 // Archive for debug?
 445                                 unlink($sImportFile);
 446 //                      }
 447
 448                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
 449
 450                         // Index file
 451                         $sThisIndexCmd = $sCMDIndex;
 452                         $fCMDStartTime = time();
 453
 454                         if (!$aResult['no-npi'])
 455                         {
 456                                 $iFileID = $oDB->getOne('select nextval(\'file\')');
 457                                 if (PEAR::isError($iFileID))
 458                                 {
 459                                         echo $iFileID->getMessage()."\n";
 460                                         exit(-1);
 461                                 }
 462                                 $sFileDir = CONST_BasePath.'/export/diff/';
 463                                 $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
 464                                 $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
 465
 466                                 if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
 467                                 $sThisIndexCmd .= $sFileDir;
 468                                 $sThisIndexCmd .= '/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT);
 469                                 $sThisIndexCmd .= ".npi.out";
 470
 471                                 preg_match('#^([0-9]{4})-([0-9]{2})-([0-9]{2})#', $sBatchEnd, $aBatchMatch);
 472                                 $sFileDir = CONST_BasePath.'/export/index/';
 473                                 $sFileDir .= $aBatchMatch[1].'/'.$aBatchMatch[2];
 474
 475                                 if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
 476                                 file_put_contents($sFileDir.'/'.$aBatchMatch[3].'.idx', "$sBatchEnd\t$iFileID\n", FILE_APPEND);
 477                         }
 478
 479                         if (!$aResult['no-index'])
 480                         {
 481                                 echo "$sThisIndexCmd\n";
 482                                 exec($sThisIndexCmd, $sJunk, $iErrorLevel);
 483                                 if ($iErrorLevel)
 484                                 {
 485                                         echo "Error: $iErrorLevel\n";
 486                                         exit($iErrorLevel);
 487                                 }
 488
 489                                 if (!$aResult['no-npi'])
 490                                 {
 491                                         $sFileDir = CONST_BasePath.'/export/diff/';
 492                                         $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
 493                                         $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
 494
 495                                         $sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out";
 496                                         echo "$sThisIndexCmd\n";
 497                                         exec($sThisIndexCmd, $sJunk, $iErrorLevel);
 498                                         if ($iErrorLevel)
 499                                         {
 500                                                 echo "Error: $iErrorLevel\n";
 501                                                 exit($iErrorLevel);
 502                                         }
 503
 504                                         rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2",
 505                                                 $sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2");
 506                                 }
 507                         }
 508
 509                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
 510                         var_Dump($sSQL);
 511                         $oDB->query($sSQL);
 512                         echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 513
 514                         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
 515                         $oDB->query($sSQL);
 516
 517                         $fDuration = time() - $fStartTime;
 518                         echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
 519                         if (!$aResult['import-osmosis-all']) exit(0);
 520
 521                         if ( CONST_Replication_Update_Interval > 60 )
 522                         {
 523                                 $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
 524                         }
 525                         else
 526                         {
 527                                 $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
 528                         }
 529                         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
 530                         sleep($iSleep);
 531                 }
 532
 533         }
 534
 535         if ($aResult['import-npi-all'])
 536         {
 537                 $iNPIID = $oDB->getOne('select max(npiid) from import_npi_log');
 538                 if (PEAR::isError($iNPIID))
 539                 {
 540                         var_dump($iNPIID);
 541                         exit(1);
 542                 }
 543                 $sConfigDirectory = CONST_InstallPath.'/settings';
 544                 $sCMDImportTemplate = CONST_InstallPath.'/nominatim/nominatim -d gazetteer -P 5433 -I -T '.CONST_BasePath.'/settings/partitionedtags.def -F ';
 545                 while(true)
 546                 {
 547                         $fStartTime = time();
 548
 549                         $iNPIID++;
 550
 551                         $sImportFile = CONST_BasePath.'/export/diff/';
 552                         $sImportFile .= str_pad(floor($iNPIID/1000000), 3, '0', STR_PAD_LEFT);
 553                         $sImportFile .= '/'.str_pad(floor($iNPIID/1000) % 1000, 3, '0', STR_PAD_LEFT);
 554                         $sImportFile .= '/'.str_pad($iNPIID % 1000, 3, '0', STR_PAD_LEFT);
 555                         $sImportFile .= ".npi";
 556                         while(!file_exists($sImportFile) && !file_exists($sImportFile.'.bz2'))
 557                         {
 558                                 echo "sleep (waiting for $sImportFile)\n";
 559                                 sleep(10);
 560                         }
 561                         if (file_exists($sImportFile.'.bz2')) $sImportFile .= '.bz2';
 562
 563                         $iFileSize = filesize($sImportFile);
 564
 565                         // Import the file
 566                         $fCMDStartTime = time();
 567                         $sCMDImport = $sCMDImportTemplate . $sImportFile;
 568                         echo $sCMDImport."\n";
 569                         exec($sCMDImport, $sJunk, $iErrorLevel);
 570                         if ($iErrorLevel)
 571                         {
 572                                 fail("Error: $iErrorLevel\n");
 573                         }
 574                         $sBatchEnd = $iNPIID;
 575                         echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 576                         $sSQL = "INSERT INTO import_npi_log values ($iNPIID, null, $iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
 577                         var_Dump($sSQL);
 578                         $oDB->query($sSQL);
 579                 }
 580
 581         }
 582
 583         function getosmosistimestamp($sOsmosisConfigDirectory)
 584         {
 585                 $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
 586                 preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
 587                 return str_replace('\:',':',$aResult[1]);
 588         }