utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  40         );
  41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  42
  43         $bDidSomething = false;
  44
  45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  47         {
  48                 if (!isset($aCMDResult['osm-file']))
  49                 {
  50                         fail('missing --osm-file for data import');
  51                 }
  52
  53                 if (!file_exists($aCMDResult['osm-file']))
  54                 {
  55                         fail('the path supplied to --osm-file does not exist');
  56                 }
  57
  58                 if (!is_readable($aCMDResult['osm-file']))
  59                 {
  60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  61                 }
  62         }
  63
  64
  65         // This is a pretty hard core default - the number of processors in the box - 1
  66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  67         if ($iInstances < 1)
  68         {
  69                 $iInstances = 1;
  70                 echo "WARNING: resetting threads to $iInstances\n";
  71         }
  72         if ($iInstances > getProcessorCount())
  73         {
  74                 $iInstances = getProcessorCount();
  75                 echo "WARNING: resetting threads to $iInstances\n";
  76         }
  77
  78         // Assume we can steal all the cache memory in the box (unless told otherwise)
  79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  80         if ($iCacheMemory > getTotalMemoryMB())
  81         {
  82                 $iCacheMemory = getCacheMemoryMB();
  83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  84         }
  85
  86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  88
  89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  90         {
  91                 echo "Create DB\n";
  92                 $bDidSomething = true;
  93                 $oDB =& DB::connect(CONST_Database_DSN, false);
  94                 if (!PEAR::isError($oDB))
  95                 {
  96                         fail('database already exists ('.CONST_Database_DSN.')');
  97                 }
  98                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  99         }
 100
 101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 102         {
 103                 echo "Setup DB\n";
 104                 $bDidSomething = true;
 105                 // TODO: path detection, detection memory, etc.
 106
 107                 $oDB =& getDB();
 108                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 109                 $pgver = (float) CONST_Postgresql_Version;
 110                 if ($pgver < 9.1) {
 111                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 112                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 113                 } else {
 114                         pgsqlRunScript('CREATE EXTENSION hstore');
 115                 }
 116                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 117                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 118                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 119                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 120                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 121                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 122                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 123                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 124                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 125                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 126         }
 127
 128         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 129         {
 130                 echo "Import\n";
 131                 $bDidSomething = true;
 132
 133                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 134                 if (!file_exists($osm2pgsql))
 135                 {
 136                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 137                         fail("osm2pgsql not found in '$osm2pgsql'");
 138                 }
 139                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 140                 $osm2pgsql .= ' -C '.$iCacheMemory;
 141                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 142                 passthruCheckReturn($osm2pgsql);
 143
 144                 $oDB =& getDB();
 145                 $x = $oDB->getRow('select * from place limit 1');
 146                 if (PEAR::isError($x)) {
 147                         fail($x->getMessage());
 148                 }
 149                 if (!$x) fail('No Data');
 150         }
 151
 152         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 153         {
 154                 echo "Functions\n";
 155                 $bDidSomething = true;
 156                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 157                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 158                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 159                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 160                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 161                 pgsqlRunScript($sTemplate);
 162         }
 163
 164         if ($aCMDResult['create-minimal-tables'])
 165         {
 166                 echo "Minimal Tables\n";
 167                 $bDidSomething = true;
 168                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 169
 170                 $sScript = '';
 171
 172                 // Backstop the import process - easliest possible import id
 173                 $sScript .= "insert into import_npi_log values (18022);\n";
 174
 175                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 176                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 177
 178                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 179                 {
 180                         list($sClass, $sType) = explode(' ', trim($sLine));
 181                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 182                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 183
 184                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 185                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 186
 187                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 188                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 189                 }
 190                 fclose($hFile);
 191                 pgsqlRunScript($sScript);
 192         }
 193
 194         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 195         {
 196                 echo "Tables\n";
 197                 $bDidSomething = true;
 198                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 199
 200                 // re-run the functions
 201                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 202                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 203                 pgsqlRunScript($sTemplate);
 204         }
 205
 206         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 207         {
 208                 echo "Partitions\n";
 209                 $bDidSomething = true;
 210                 $oDB =& getDB();
 211                 $sSQL = 'select partition from country_name order by country_code';
 212                 $aPartitions = $oDB->getCol($sSQL);
 213                 if (PEAR::isError($aPartitions))
 214                 {
 215                         fail($aPartitions->getMessage());
 216                 }
 217                 $aPartitions[] = 0;
 218
 219                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 220                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 221                 foreach($aMatches as $aMatch)
 222                 {
 223                         $sResult = '';
 224                         foreach($aPartitions as $sPartitionName)
 225                         {
 226                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 227                         }
 228                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 229                 }
 230
 231                 pgsqlRunScript($sTemplate);
 232         }
 233
 234         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 235         {
 236                 $bDidSomething = true;
 237                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 238                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 239                 if (file_exists($sWikiArticlesFile))
 240                 {
 241                         echo "Importing wikipedia articles...";
 242                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 243                         echo "...done\n";
 244                 }
 245                 else
 246                 {
 247                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 248                 }
 249                 if (file_exists($sWikiRedirectsFile))
 250                 {
 251                         echo "Importing wikipedia redirects...";
 252                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 253                         echo "...done\n";
 254                 }
 255                 else
 256                 {
 257                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 258                 }
 259         }
 260
 261
 262         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 263         {
 264                 echo "Drop old Data\n";
 265                 $bDidSomething = true;
 266
 267                 $oDB =& getDB();
 268                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 269                 echo '.';
 270                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 271                 echo '.';
 272                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 273                 echo '.';
 274                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 275                 echo '.';
 276                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 277                 echo '.';
 278                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 279                 echo '.';
 280                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 281                 echo '.';
 282                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 283                 echo '.';
 284                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 285                 echo '.';
 286
 287                 $sSQL = 'select partition from country_name order by country_code';
 288                 $aPartitions = $oDB->getCol($sSQL);
 289                 if (PEAR::isError($aPartitions))
 290                 {
 291                         fail($aPartitions->getMessage());
 292                 }
 293                 $aPartitions[] = 0;
 294                 foreach($aPartitions as $sPartition)
 295                 {
 296                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 297                         echo '.';
 298                 }
 299
 300                 // used by getorcreate_word_id to ignore frequent partial words
 301                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 302                 echo ".\n";
 303
 304                 // pre-create the word list
 305                 if (!$aCMDResult['disable-token-precalc'])
 306                 {
 307                         echo "Loading word list\n";
 308                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 309                 }
 310
 311                 echo "Load Data\n";
 312                 $aDBInstances = array();
 313                 for($i = 0; $i < $iInstances; $i++)
 314                 {
 315                         $aDBInstances[$i] =& getDB(true);
 316                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 317                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 318                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 319                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 320                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 321                 }
 322                 $bAnyBusy = true;
 323                 while($bAnyBusy)
 324                 {
 325                         $bAnyBusy = false;
 326                         for($i = 0; $i < $iInstances; $i++)
 327                         {
 328                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 329                         }
 330                         sleep(1);
 331                         echo '.';
 332                 }
 333                 echo "\n";
 334                 echo "Reanalysing database...\n";
 335                 pgsqlRunScript('ANALYSE');
 336         }
 337
 338         if ($aCMDResult['create-roads'])
 339         {
 340                 $bDidSomething = true;
 341
 342                 $oDB =& getDB();
 343                 $aDBInstances = array();
 344                 for($i = 0; $i < $iInstances; $i++)
 345                 {
 346                         $aDBInstances[$i] =& getDB(true);
 347                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 348                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 349                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 350                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 351                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 352                 }
 353                 $bAnyBusy = true;
 354                 while($bAnyBusy)
 355                 {
 356                         $bAnyBusy = false;
 357                         for($i = 0; $i < $iInstances; $i++)
 358                         {
 359                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 360                         }
 361                         sleep(1);
 362                         echo '.';
 363                 }
 364                 echo "\n";
 365         }
 366
 367         if ($aCMDResult['import-tiger-data'])
 368         {
 369                 $bDidSomething = true;
 370
 371                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 372
 373                 $aDBInstances = array();
 374                 for($i = 0; $i < $iInstances; $i++)
 375                 {
 376                         $aDBInstances[$i] =& getDB(true);
 377                 }
 378
 379                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 380                 {
 381                         echo $sFile.': ';
 382                         $hFile = fopen($sFile, "r");
 383                         $sSQL = fgets($hFile, 100000);
 384                         $iLines = 0;
 385
 386                         while(true)
 387                         {
 388                                 for($i = 0; $i < $iInstances; $i++)
 389                                 {
 390                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 391                                         {
 392                                                 while(pg_get_result($aDBInstances[$i]->connection));
 393                                                 $sSQL = fgets($hFile, 100000);
 394                                                 if (!$sSQL) break 2;
 395                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 396                                                 $iLines++;
 397                                                 if ($iLines == 1000)
 398                                                 {
 399                                                         echo ".";
 400                                                         $iLines = 0;
 401                                                 }
 402                                         }
 403                                 }
 404                                 usleep(10);
 405                         }
 406
 407                         fclose($hFile);
 408
 409                         $bAnyBusy = true;
 410                         while($bAnyBusy)
 411                         {
 412                                 $bAnyBusy = false;
 413                                 for($i = 0; $i < $iInstances; $i++)
 414                                 {
 415                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 416                                 }
 417                                 usleep(10);
 418                         }
 419                         echo "\n";
 420                 }
 421
 422                 echo "Creating indexes\n";
 423                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 424         }
 425
 426         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 427         {
 428                 $bDidSomething = true;
 429                 $oDB =& getDB();
 430                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 431                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 432                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 433                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 434                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 435                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 436                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 437
 438                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 439                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 440                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 441                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 442         }
 443
 444         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 445         {
 446                 $bDidSomething = true;
 447                 $oDB =& getDB();
 448
 449                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 450                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 451                 {
 452                         echo "settings/configuration.txt already exists\n";
 453                 }
 454                 else
 455                 {
 456                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 457                         // server layout changed afer license change, fix path to minutely diffs
 458                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 459                 }
 460
 461                 // Find the last node in the DB
 462                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
 463
 464                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 465                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
 466                 $sLastNodeXML = file_get_contents($sLastNodeURL);
 467                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 468                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 469
 470
 471                 // Search for the correct state file - uses file timestamps
 472                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
 473                 $sRep = file_get_contents($sRepURL);
 474                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 475                 $aPrevRepMatch = false;
 476                 foreach($aRepMatches as $aRepMatch)
 477                 {
 478                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 479                         $aPrevRepMatch = $aRepMatch;
 480                 }
 481                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 482
 483                 $sRepURL .= $aRepMatch[1];
 484                 $sRep = file_get_contents($sRepURL);
 485                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 486                 $aPrevRepMatch = false;
 487                 foreach($aRepMatches as $aRepMatch)
 488                 {
 489                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 490                         $aPrevRepMatch = $aRepMatch;
 491                 }
 492                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 493
 494                 $sRepURL .= $aRepMatch[1];
 495                 $sRep = file_get_contents($sRepURL);
 496                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 497                 $aPrevRepMatch = false;
 498                 foreach($aRepMatches as $aRepMatch)
 499                 {
 500                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 501                         $aPrevRepMatch = $aRepMatch;
 502                 }
 503                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 504
 505                 $sRepURL .= $aRepMatch[1].'.state.txt';
 506                 echo "Getting state file: $sRepURL\n";
 507                 $sStateFile = file_get_contents($sRepURL);
 508                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 509                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 510                 echo "Updating DB status\n";
 511                 pg_query($oDB->connection, 'TRUNCATE import_status');
 512                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 513                 pg_query($oDB->connection, $sSQL);
 514         }
 515
 516         if ($aCMDResult['index'] || $aCMDResult['all'])
 517         {
 518                 $bDidSomething = true;
 519                 $sOutputFile = '';
 520                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 521                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 522                 passthruCheckReturn($sBaseCmd.' -R 4');
 523                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 524                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 525                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 526                 passthruCheckReturn($sBaseCmd.' -r 26');
 527         }
 528
 529         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 530         {
 531                 echo "Search indices\n";
 532                 $bDidSomething = true;
 533                 $oDB =& getDB();
 534                 $sSQL = 'select partition from country_name order by country_code';
 535                 $aPartitions = $oDB->getCol($sSQL);
 536                 if (PEAR::isError($aPartitions))
 537                 {
 538                         fail($aPartitions->getMessage());
 539                 }
 540                 $aPartitions[] = 0;
 541
 542                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 543                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 544                 foreach($aMatches as $aMatch)
 545                 {
 546                         $sResult = '';
 547                         foreach($aPartitions as $sPartitionName)
 548                         {
 549                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 550                         }
 551                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 552                 }
 553
 554                 pgsqlRunScript($sTemplate);
 555         }
 556
 557         if (isset($aCMDResult['create-website']))
 558         {
 559                 $bDidSomething = true;
 560                 $sTargetDir = $aCMDResult['create-website'];
 561                 if (!is_dir($sTargetDir))
 562                 {
 563                         echo "You must create the website directory before calling this function.\n";
 564                         fail("Target directory does not exist.");
 565                 }
 566
 567                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 568                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 569                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 570                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 571                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 572                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 573                 echo "Symlinks created\n";
 574         }
 575
 576         if (!$bDidSomething)
 577         {
 578                 showUsage($aCMDOptions, true);
 579         }
 580
 581         function pgsqlRunScriptFile($sFilename)
 582         {
 583                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 584
 585                 // Convert database DSN to psql parameters
 586                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 587                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 588                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 589
 590                 $aDescriptors = array(
 591                         0 => array('pipe', 'r'),
 592                         1 => array('pipe', 'w'),
 593                         2 => array('file', '/dev/null', 'a')
 594                 );
 595                 $ahPipes = null;
 596                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 597                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 598
 599                 fclose($ahPipes[0]);
 600
 601                 // TODO: error checking
 602                 while(!feof($ahPipes[1]))
 603                 {
 604                         echo fread($ahPipes[1], 4096);
 605                 }
 606                 fclose($ahPipes[1]);
 607
 608                 proc_close($hProcess);
 609         }
 610
 611         function pgsqlRunScript($sScript)
 612         {
 613                 // Convert database DSN to psql parameters
 614                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 615                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 616                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 617                 $aDescriptors = array(
 618                         0 => array('pipe', 'r'),
 619                         1 => STDOUT,
 620                         2 => STDERR
 621                 );
 622                 $ahPipes = null;
 623                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 624                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 625
 626                 while(strlen($sScript))
 627                 {
 628                         $written = fwrite($ahPipes[0], $sScript);
 629                         $sScript = substr($sScript, $written);
 630                 }
 631                 fclose($ahPipes[0]);
 632                 proc_close($hProcess);
 633         }
 634
 635         function pgsqlRunRestoreData($sDumpFile)
 636         {
 637                 // Convert database DSN to psql parameters
 638                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 639                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 640                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 641
 642                 $aDescriptors = array(
 643                         0 => array('pipe', 'r'),
 644                         1 => array('pipe', 'w'),
 645                         2 => array('file', '/dev/null', 'a')
 646                 );
 647                 $ahPipes = null;
 648                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 649                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 650
 651                 fclose($ahPipes[0]);
 652
 653                 // TODO: error checking
 654                 while(!feof($ahPipes[1]))
 655                 {
 656                         echo fread($ahPipes[1], 4096);
 657                 }
 658                 fclose($ahPipes[1]);
 659
 660                 proc_close($hProcess);
 661         }
 662
 663         function pgsqlRunDropAndRestore($sDumpFile)
 664         {
 665                 // Convert database DSN to psql parameters
 666                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 667                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 668                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 669
 670                 $aDescriptors = array(
 671                         0 => array('pipe', 'r'),
 672                         1 => array('pipe', 'w'),
 673                         2 => array('file', '/dev/null', 'a')
 674                 );
 675                 $ahPipes = null;
 676                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 677                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 678
 679                 fclose($ahPipes[0]);
 680
 681                 // TODO: error checking
 682                 while(!feof($ahPipes[1]))
 683                 {
 684                         echo fread($ahPipes[1], 4096);
 685                 }
 686                 fclose($ahPipes[1]);
 687
 688                 proc_close($hProcess);
 689         }
 690
 691         function passthruCheckReturn($cmd)
 692         {
 693                 $result = -1;
 694                 passthru($cmd, $result);
 695                 if ($result != 0) fail('Error executing external command: '.$cmd);
 696         }