utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  40         );
  41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  42
  43         $bDidSomething = false;
  44
  45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  47         {
  48                 if (!isset($aCMDResult['osm-file']))
  49                 {
  50                         fail('missing --osm-file for data import');
  51                 }
  52
  53                 if (!file_exists($aCMDResult['osm-file']))
  54                 {
  55                         fail('the path supplied to --osm-file does not exist');
  56                 }
  57
  58                 if (!is_readable($aCMDResult['osm-file']))
  59                 {
  60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  61                 }
  62         }
  63
  64
  65         // This is a pretty hard core default - the number of processors in the box - 1
  66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  67         if ($iInstances < 1)
  68         {
  69                 $iInstances = 1;
  70                 echo "WARNING: resetting threads to $iInstances\n";
  71         }
  72         if ($iInstances > getProcessorCount())
  73         {
  74                 $iInstances = getProcessorCount();
  75                 echo "WARNING: resetting threads to $iInstances\n";
  76         }
  77
  78         // Assume we can steal all the cache memory in the box (unless told otherwise)
  79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  80         if ($iCacheMemory > getTotalMemoryMB())
  81         {
  82                 $iCacheMemory = getCacheMemoryMB();
  83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  84         }
  85
  86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  88
  89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  90         {
  91                 echo "Create DB\n";
  92                 $bDidSomething = true;
  93                 $oDB =& DB::connect(CONST_Database_DSN, false);
  94                 if (!PEAR::isError($oDB))
  95                 {
  96                         fail('database already exists ('.CONST_Database_DSN.')');
  97                 }
  98                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  99         }
 100
 101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 102         {
 103                 echo "Setup DB\n";
 104                 $bDidSomething = true;
 105                 // TODO: path detection, detection memory, etc.
 106
 107                 $oDB =& getDB();
 108                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 109                 $pgver = (float) CONST_Postgresql_Version;
 110                 if ($pgver < 9.1) {
 111                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 112                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 113                 } else {
 114                         pgsqlRunScript('CREATE EXTENSION hstore');
 115                 }
 116                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 117                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 118                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 119                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 120                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 121                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 122                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 123                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 124                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 125                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 126         }
 127
 128         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 129         {
 130                 echo "Import\n";
 131                 $bDidSomething = true;
 132
 133                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 134                 if (!file_exists($osm2pgsql))
 135                 {
 136                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 137                         fail("osm2pgsql not found in '$osm2pgsql'");
 138                 }
 139                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 140                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 141                 $osm2pgsql .= ' -C 16000';
 142                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 143                 passthruCheckReturn($osm2pgsql);
 144
 145                 $oDB =& getDB();
 146                 $x = $oDB->getRow('select * from place limit 1');
 147                 if (PEAR::isError($x)) {
 148                         fail($x->getMessage());
 149                 }
 150                 if (!$x) fail('No Data');
 151         }
 152
 153         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 154         {
 155                 echo "Functions\n";
 156                 $bDidSomething = true;
 157                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 158                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 159                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 160                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 161                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 162                 pgsqlRunScript($sTemplate);
 163         }
 164
 165         if ($aCMDResult['create-minimal-tables'])
 166         {
 167                 echo "Minimal Tables\n";
 168                 $bDidSomething = true;
 169                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 170
 171                 $sScript = '';
 172
 173                 // Backstop the import process - easliest possible import id
 174                 $sScript .= "insert into import_npi_log values (18022);\n";
 175
 176                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 177                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 178
 179                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 180                 {
 181                         list($sClass, $sType) = explode(' ', trim($sLine));
 182                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 183                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 184
 185                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 186                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 187
 188                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 189                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 190                 }
 191                 fclose($hFile);
 192                 pgsqlRunScript($sScript);
 193         }
 194
 195         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 196         {
 197                 echo "Tables\n";
 198                 $bDidSomething = true;
 199                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 200
 201                 // re-run the functions
 202                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 203                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 204                 pgsqlRunScript($sTemplate);
 205         }
 206
 207         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 208         {
 209                 echo "Partitions\n";
 210                 $bDidSomething = true;
 211                 $oDB =& getDB();
 212                 $sSQL = 'select partition from country_name order by country_code';
 213                 $aPartitions = $oDB->getCol($sSQL);
 214                 if (PEAR::isError($aPartitions))
 215                 {
 216                         fail($aPartitions->getMessage());
 217                 }
 218                 $aPartitions[] = 0;
 219
 220                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 221                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 222                 foreach($aMatches as $aMatch)
 223                 {
 224                         $sResult = '';
 225                         foreach($aPartitions as $sPartitionName)
 226                         {
 227                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 228                         }
 229                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 230                 }
 231
 232                 pgsqlRunScript($sTemplate);
 233         }
 234
 235         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 236         {
 237                 $bDidSomething = true;
 238                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 239                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 240                 if (file_exists($sWikiArticlesFile))
 241                 {
 242                         echo "Importing wikipedia articles...";
 243                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 244                         echo "...done\n";
 245                 }
 246                 else
 247                 {
 248                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 249                 }
 250                 if (file_exists($sWikiRedirectsFile))
 251                 {
 252                         echo "Importing wikipedia redirects...";
 253                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 254                         echo "...done\n";
 255                 }
 256                 else
 257                 {
 258                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 259                 }
 260         }
 261
 262
 263         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 264         {
 265                 echo "Drop old Data\n";
 266                 $bDidSomething = true;
 267
 268                 $oDB =& getDB();
 269                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 270                 echo '.';
 271                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 272                 echo '.';
 273                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 274                 echo '.';
 275                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 276                 echo '.';
 277                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 278                 echo '.';
 279                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 280                 echo '.';
 281                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 282                 echo '.';
 283                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 284                 echo '.';
 285                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 286                 echo '.';
 287
 288                 $sSQL = 'select partition from country_name order by country_code';
 289                 $aPartitions = $oDB->getCol($sSQL);
 290                 if (PEAR::isError($aPartitions))
 291                 {
 292                         fail($aPartitions->getMessage());
 293                 }
 294                 $aPartitions[] = 0;
 295                 foreach($aPartitions as $sPartition)
 296                 {
 297                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 298                         echo '.';
 299                 }
 300
 301                 // used by getorcreate_word_id to ignore frequent partial words
 302                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 303                 echo ".\n";
 304
 305                 // pre-create the word list
 306                 if (!$aCMDResult['disable-token-precalc'])
 307                 {
 308                         echo "Loading word list\n";
 309                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 310                 }
 311
 312                 echo "Load Data\n";
 313                 $aDBInstances = array();
 314                 for($i = 0; $i < $iInstances; $i++)
 315                 {
 316                         $aDBInstances[$i] =& getDB(true);
 317                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 318                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 319                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 320                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 321                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 322                 }
 323                 $bAnyBusy = true;
 324                 while($bAnyBusy)
 325                 {
 326                         $bAnyBusy = false;
 327                         for($i = 0; $i < $iInstances; $i++)
 328                         {
 329                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 330                         }
 331                         sleep(1);
 332                         echo '.';
 333                 }
 334                 echo "\n";
 335                 echo "Reanalysing database...\n";
 336                 pgsqlRunScript('ANALYSE');
 337         }
 338
 339         if ($aCMDResult['create-roads'])
 340         {
 341                 $bDidSomething = true;
 342
 343                 $oDB =& getDB();
 344                 $aDBInstances = array();
 345                 for($i = 0; $i < $iInstances; $i++)
 346                 {
 347                         $aDBInstances[$i] =& getDB(true);
 348                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 349                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 350                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 351                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 352                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 353                 }
 354                 $bAnyBusy = true;
 355                 while($bAnyBusy)
 356                 {
 357                         $bAnyBusy = false;
 358                         for($i = 0; $i < $iInstances; $i++)
 359                         {
 360                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 361                         }
 362                         sleep(1);
 363                         echo '.';
 364                 }
 365                 echo "\n";
 366         }
 367
 368         if ($aCMDResult['import-tiger-data'])
 369         {
 370                 $bDidSomething = true;
 371
 372                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 373
 374                 $aDBInstances = array();
 375                 for($i = 0; $i < $iInstances; $i++)
 376                 {
 377                         $aDBInstances[$i] =& getDB(true);
 378                 }
 379
 380                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 381                 {
 382                         echo $sFile.': ';
 383                         $hFile = fopen($sFile, "r");
 384                         $sSQL = fgets($hFile, 100000);
 385                         $iLines = 0;
 386
 387                         while(true)
 388                         {
 389                                 for($i = 0; $i < $iInstances; $i++)
 390                                 {
 391                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 392                                         {
 393                                                 while(pg_get_result($aDBInstances[$i]->connection));
 394                                                 $sSQL = fgets($hFile, 100000);
 395                                                 if (!$sSQL) break 2;
 396                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 397                                                 $iLines++;
 398                                                 if ($iLines == 1000)
 399                                                 {
 400                                                         echo ".";
 401                                                         $iLines = 0;
 402                                                 }
 403                                         }
 404                                 }
 405                                 usleep(10);
 406                         }
 407
 408                         fclose($hFile);
 409
 410                         $bAnyBusy = true;
 411                         while($bAnyBusy)
 412                         {
 413                                 $bAnyBusy = false;
 414                                 for($i = 0; $i < $iInstances; $i++)
 415                                 {
 416                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 417                                 }
 418                                 usleep(10);
 419                         }
 420                         echo "\n";
 421                 }
 422
 423                 echo "Creating indexes\n";
 424                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 425         }
 426
 427         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 428         {
 429                 $bDidSomething = true;
 430                 $oDB =& getDB();
 431                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 432                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 433                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 434                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 435                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 436                 $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";
 437                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 438
 439                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 440                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 441                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 442                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 443         }
 444
 445         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 446         {
 447                 $bDidSomething = true;
 448                 $oDB =& getDB();
 449
 450                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 451                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 452                 {
 453                         echo "settings/configuration.txt already exists\n";
 454                 }
 455                 else
 456                 {
 457                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 458                         // server layout changed afer license change, fix path to minutely diffs
 459                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 460                 }
 461
 462                 // Find the last node in the DB
 463                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
 464
 465                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 466                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
 467                 $sLastNodeXML = file_get_contents($sLastNodeURL);
 468                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 469                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 470
 471
 472                 // Search for the correct state file - uses file timestamps
 473                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
 474                 $sRep = file_get_contents($sRepURL);
 475                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 476                 $aPrevRepMatch = false;
 477                 foreach($aRepMatches as $aRepMatch)
 478                 {
 479                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 480                         $aPrevRepMatch = $aRepMatch;
 481                 }
 482                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 483
 484                 $sRepURL .= $aRepMatch[1];
 485                 $sRep = file_get_contents($sRepURL);
 486                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 487                 $aPrevRepMatch = false;
 488                 foreach($aRepMatches as $aRepMatch)
 489                 {
 490                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 491                         $aPrevRepMatch = $aRepMatch;
 492                 }
 493                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 494
 495                 $sRepURL .= $aRepMatch[1];
 496                 $sRep = file_get_contents($sRepURL);
 497                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 498                 $aPrevRepMatch = false;
 499                 foreach($aRepMatches as $aRepMatch)
 500                 {
 501                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 502                         $aPrevRepMatch = $aRepMatch;
 503                 }
 504                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 505
 506                 $sRepURL .= $aRepMatch[1].'.state.txt';
 507                 echo "Getting state file: $sRepURL\n";
 508                 $sStateFile = file_get_contents($sRepURL);
 509                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 510                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 511                 echo "Updating DB status\n";
 512                 pg_query($oDB->connection, 'TRUNCATE import_status');
 513                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 514                 pg_query($oDB->connection, $sSQL);
 515         }
 516
 517         if ($aCMDResult['index'] || $aCMDResult['all'])
 518         {
 519                 $bDidSomething = true;
 520                 $sOutputFile = '';
 521                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 522                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 523                 passthruCheckReturn($sBaseCmd.' -R 4');
 524                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 525                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 526                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 527                 passthruCheckReturn($sBaseCmd.' -r 26');
 528         }
 529
 530         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 531         {
 532                 echo "Search indices\n";
 533                 $bDidSomething = true;
 534                 $oDB =& getDB();
 535                 $sSQL = 'select partition from country_name order by country_code';
 536                 $aPartitions = $oDB->getCol($sSQL);
 537                 if (PEAR::isError($aPartitions))
 538                 {
 539                         fail($aPartitions->getMessage());
 540                 }
 541                 $aPartitions[] = 0;
 542
 543                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 544                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 545                 foreach($aMatches as $aMatch)
 546                 {
 547                         $sResult = '';
 548                         foreach($aPartitions as $sPartitionName)
 549                         {
 550                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 551                         }
 552                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 553                 }
 554
 555                 pgsqlRunScript($sTemplate);
 556         }
 557
 558         if (isset($aCMDResult['create-website']))
 559         {
 560                 $bDidSomething = true;
 561                 $sTargetDir = $aCMDResult['create-website'];
 562                 if (!is_dir($sTargetDir))
 563                 {
 564                         echo "You must create the website directory before calling this function.\n";
 565                         fail("Target directory does not exist.");
 566                 }
 567
 568                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 569                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 570                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 571                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 572                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 573                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 574                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 575                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 576                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 577                 echo "Symlinks created\n";
 578         }
 579
 580         if (!$bDidSomething)
 581         {
 582                 showUsage($aCMDOptions, true);
 583         }
 584
 585         function pgsqlRunScriptFile($sFilename)
 586         {
 587                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 588
 589                 // Convert database DSN to psql parameters
 590                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 591                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 592                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 593
 594                 $aDescriptors = array(
 595                         0 => array('pipe', 'r'),
 596                         1 => array('pipe', 'w'),
 597                         2 => array('file', '/dev/null', 'a')
 598                 );
 599                 $ahPipes = null;
 600                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 601                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 602
 603                 fclose($ahPipes[0]);
 604
 605                 // TODO: error checking
 606                 while(!feof($ahPipes[1]))
 607                 {
 608                         echo fread($ahPipes[1], 4096);
 609                 }
 610                 fclose($ahPipes[1]);
 611
 612                 proc_close($hProcess);
 613         }
 614
 615         function pgsqlRunScript($sScript)
 616         {
 617                 // Convert database DSN to psql parameters
 618                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 619                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 620                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 621                 $aDescriptors = array(
 622                         0 => array('pipe', 'r'),
 623                         1 => STDOUT,
 624                         2 => STDERR
 625                 );
 626                 $ahPipes = null;
 627                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 628                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 629
 630                 while(strlen($sScript))
 631                 {
 632                         $written = fwrite($ahPipes[0], $sScript);
 633                         $sScript = substr($sScript, $written);
 634                 }
 635                 fclose($ahPipes[0]);
 636                 proc_close($hProcess);
 637         }
 638
 639         function pgsqlRunRestoreData($sDumpFile)
 640         {
 641                 // Convert database DSN to psql parameters
 642                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 643                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 644                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 645
 646                 $aDescriptors = array(
 647                         0 => array('pipe', 'r'),
 648                         1 => array('pipe', 'w'),
 649                         2 => array('file', '/dev/null', 'a')
 650                 );
 651                 $ahPipes = null;
 652                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 653                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 654
 655                 fclose($ahPipes[0]);
 656
 657                 // TODO: error checking
 658                 while(!feof($ahPipes[1]))
 659                 {
 660                         echo fread($ahPipes[1], 4096);
 661                 }
 662                 fclose($ahPipes[1]);
 663
 664                 proc_close($hProcess);
 665         }
 666
 667         function pgsqlRunDropAndRestore($sDumpFile)
 668         {
 669                 // Convert database DSN to psql parameters
 670                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 671                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 672                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 673
 674                 $aDescriptors = array(
 675                         0 => array('pipe', 'r'),
 676                         1 => array('pipe', 'w'),
 677                         2 => array('file', '/dev/null', 'a')
 678                 );
 679                 $ahPipes = null;
 680                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 681                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 682
 683                 fclose($ahPipes[0]);
 684
 685                 // TODO: error checking
 686                 while(!feof($ahPipes[1]))
 687                 {
 688                         echo fread($ahPipes[1], 4096);
 689                 }
 690                 fclose($ahPipes[1]);
 691
 692                 proc_close($hProcess);
 693         }
 694
 695         function passthruCheckReturn($cmd)
 696         {
 697                 $result = -1;
 698                 passthru($cmd, $result);
 699                 if ($result != 0) fail('Error executing external command: '.$cmd);
 700         }