utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
  28                 array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
  29                 array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
  30                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  31                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  32                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  33                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  34                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  35                 array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
  36                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  38                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  39                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  40                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  41                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  42         );
  43         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  44
  45         $bDidSomething = false;
  46
  47         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  48         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  49         {
  50                 if (!isset($aCMDResult['osm-file']))
  51                 {
  52                         fail('missing --osm-file for data import');
  53                 }
  54
  55                 if (!file_exists($aCMDResult['osm-file']))
  56                 {
  57                         fail('the path supplied to --osm-file does not exist');
  58                 }
  59
  60                 if (!is_readable($aCMDResult['osm-file']))
  61                 {
  62                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  63                 }
  64         }
  65
  66
  67         // This is a pretty hard core default - the number of processors in the box - 1
  68         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  69         if ($iInstances < 1)
  70         {
  71                 $iInstances = 1;
  72                 echo "WARNING: resetting threads to $iInstances\n";
  73         }
  74         if ($iInstances > getProcessorCount())
  75         {
  76                 $iInstances = getProcessorCount();
  77                 echo "WARNING: resetting threads to $iInstances\n";
  78         }
  79
  80         // Assume we can steal all the cache memory in the box (unless told otherwise)
  81         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  82         if ($iCacheMemory > getTotalMemoryMB())
  83         {
  84                 $iCacheMemory = getCacheMemoryMB();
  85                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  86         }
  87
  88         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  89         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  90
  91         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  92         {
  93                 echo "Create DB\n";
  94                 $bDidSomething = true;
  95                 $oDB =& DB::connect(CONST_Database_DSN, false);
  96                 if (!PEAR::isError($oDB))
  97                 {
  98                         fail('database already exists ('.CONST_Database_DSN.')');
  99                 }
 100                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 101         }
 102
 103         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 104         {
 105                 echo "Setup DB\n";
 106                 $bDidSomething = true;
 107                 // TODO: path detection, detection memory, etc.
 108
 109                 $oDB =& getDB();
 110
 111                 $sVersionString = $oDB->getOne('select version()');
 112                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 113                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 114                 {
 115                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 116                         exit;
 117                 }
 118
 119                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 124                 } else {
 125                         pgsqlRunScript('CREATE EXTENSION hstore');
 126                 }
 127
 128                 $fPostgisVersion = (float) CONST_Postgis_Version;
 129                 if ($fPostgisVersion < 2.0) {
 130                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 131                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 132                 } else {
 133                         pgsqlRunScript('CREATE EXTENSION postgis');
 134                 }
 135                 if ($fPostgisVersion < 2.1) {
 136                         // Function was renamed in 2.1 and throws an annoying deprecation warning
 137                         pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
 138                 }
 139                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 140                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 141                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 142                 {
 143                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 144                         exit;
 145                 }
 146
 147                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 148                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 149                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 150                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
 151                 if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz'))
 152                 {
 153                         pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
 154                 }
 155                 else
 156                 {
 157                         echo "WARNING: external UK postcode table not found.\n";
 158                 }
 159                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 160                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 161                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 162
 163                 if ($aCMDResult['no-partitions'])
 164                 {
 165                         pgsqlRunScript('update country_name set partition = 0');
 166                 }
 167
 168                 // the following will be needed by create_functions later but
 169                 // is only defined in the subsequently called create_tables.
 170                 // Create dummies here that will be overwritten by the proper
 171                 // versions in create-tables.
 172                 pgsqlRunScript('CREATE TABLE place_boundingbox ()');
 173                 pgsqlRunScript('create type wikipedia_article_match as ()');
 174         }
 175
 176         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 177         {
 178                 echo "Import\n";
 179                 $bDidSomething = true;
 180
 181                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 182                 if (!file_exists($osm2pgsql))
 183                 {
 184                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 185                         fail("osm2pgsql not found in '$osm2pgsql'");
 186                 }
 187
 188                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 189                 {
 190                         $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 191                 }
 192                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 193                 $osm2pgsql .= ' -C '.$iCacheMemory;
 194                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 195                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 196                 passthruCheckReturn($osm2pgsql);
 197
 198                 $oDB =& getDB();
 199                 $x = $oDB->getRow('select * from place limit 1');
 200                 if (PEAR::isError($x)) {
 201                         fail($x->getMessage());
 202                 }
 203                 if (!$x) fail('No Data');
 204         }
 205
 206         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 207         {
 208                 echo "Functions\n";
 209                 $bDidSomething = true;
 210                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 211                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 212                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 213                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 214                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 215                 if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
 216                 pgsqlRunScript($sTemplate);
 217         }
 218
 219         if ($aCMDResult['create-minimal-tables'])
 220         {
 221                 echo "Minimal Tables\n";
 222                 $bDidSomething = true;
 223                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 224
 225                 $sScript = '';
 226
 227                 // Backstop the import process - easliest possible import id
 228                 $sScript .= "insert into import_npi_log values (18022);\n";
 229
 230                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 231                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 232
 233                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 234                 {
 235                         list($sClass, $sType) = explode(' ', trim($sLine));
 236                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 237                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 238
 239                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 240                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 241
 242                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 243                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 244                 }
 245                 fclose($hFile);
 246                 pgsqlRunScript($sScript);
 247         }
 248
 249         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 250         {
 251                 echo "Tables\n";
 252                 $bDidSomething = true;
 253                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 254
 255                 // re-run the functions
 256                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 257                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 258                 pgsqlRunScript($sTemplate);
 259         }
 260
 261         if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 262         {
 263                 echo "Partition Tables\n";
 264                 $bDidSomething = true;
 265                 $oDB =& getDB();
 266                 $sSQL = 'select distinct partition from country_name';
 267                 $aPartitions = $oDB->getCol($sSQL);
 268                 if (PEAR::isError($aPartitions))
 269                 {
 270                         fail($aPartitions->getMessage());
 271                 }
 272                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 273
 274                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
 275                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 276                 foreach($aMatches as $aMatch)
 277                 {
 278                         $sResult = '';
 279                         foreach($aPartitions as $sPartitionName)
 280                         {
 281                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 282                         }
 283                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 284                 }
 285
 286                 pgsqlRunScript($sTemplate);
 287         }
 288
 289
 290         if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
 291         {
 292                 echo "Partition Functions\n";
 293                 $bDidSomething = true;
 294                 $oDB =& getDB();
 295                 $sSQL = 'select distinct partition from country_name';
 296                 $aPartitions = $oDB->getCol($sSQL);
 297                 if (PEAR::isError($aPartitions))
 298                 {
 299                         fail($aPartitions->getMessage());
 300                 }
 301                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 302
 303                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 304                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 305                 foreach($aMatches as $aMatch)
 306                 {
 307                         $sResult = '';
 308                         foreach($aPartitions as $sPartitionName)
 309                         {
 310                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 311                         }
 312                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 313                 }
 314
 315                 pgsqlRunScript($sTemplate);
 316         }
 317
 318         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 319         {
 320                 $bDidSomething = true;
 321                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 322                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 323                 if (file_exists($sWikiArticlesFile))
 324                 {
 325                         echo "Importing wikipedia articles...";
 326                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 327                         echo "...done\n";
 328                 }
 329                 else
 330                 {
 331                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 332                 }
 333                 if (file_exists($sWikiRedirectsFile))
 334                 {
 335                         echo "Importing wikipedia redirects...";
 336                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 337                         echo "...done\n";
 338                 }
 339                 else
 340                 {
 341                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 342                 }
 343         }
 344
 345
 346         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 347         {
 348                 echo "Drop old Data\n";
 349                 $bDidSomething = true;
 350
 351                 $oDB =& getDB();
 352                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 353                 echo '.';
 354                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 355                 echo '.';
 356                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 357                 echo '.';
 358                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 359                 echo '.';
 360                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 361                 echo '.';
 362                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 363                 echo '.';
 364                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 365                 echo '.';
 366                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 367                 echo '.';
 368                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 369                 echo '.';
 370
 371                 $sSQL = 'select distinct partition from country_name';
 372                 $aPartitions = $oDB->getCol($sSQL);
 373                 if (PEAR::isError($aPartitions))
 374                 {
 375                         fail($aPartitions->getMessage());
 376                 }
 377                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 378                 foreach($aPartitions as $sPartition)
 379                 {
 380                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 381                         echo '.';
 382                 }
 383
 384                 // used by getorcreate_word_id to ignore frequent partial words
 385                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 386                 echo ".\n";
 387
 388                 // pre-create the word list
 389                 if (!$aCMDResult['disable-token-precalc'])
 390                 {
 391                         echo "Loading word list\n";
 392                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 393                 }
 394
 395                 echo "Load Data\n";
 396                 $aDBInstances = array();
 397                 for($i = 0; $i < $iInstances; $i++)
 398                 {
 399                         $aDBInstances[$i] =& getDB(true);
 400                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 401                         $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 402                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 403                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 404                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 405                 }
 406                 $bAnyBusy = true;
 407                 while($bAnyBusy)
 408                 {
 409                         $bAnyBusy = false;
 410                         for($i = 0; $i < $iInstances; $i++)
 411                         {
 412                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 413                         }
 414                         sleep(1);
 415                         echo '.';
 416                 }
 417                 echo "\n";
 418                 echo "Reanalysing database...\n";
 419                 pgsqlRunScript('ANALYSE');
 420         }
 421
 422         if ($aCMDResult['create-roads'])
 423         {
 424                 $bDidSomething = true;
 425
 426                 $oDB =& getDB();
 427                 $aDBInstances = array();
 428                 for($i = 0; $i < $iInstances; $i++)
 429                 {
 430                         $aDBInstances[$i] =& getDB(true);
 431                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 432                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 433                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 434                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 435                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 436                 }
 437                 $bAnyBusy = true;
 438                 while($bAnyBusy)
 439                 {
 440                         $bAnyBusy = false;
 441                         for($i = 0; $i < $iInstances; $i++)
 442                         {
 443                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 444                         }
 445                         sleep(1);
 446                         echo '.';
 447                 }
 448                 echo "\n";
 449         }
 450
 451         if ($aCMDResult['import-tiger-data'])
 452         {
 453                 $bDidSomething = true;
 454
 455                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 456
 457                 $aDBInstances = array();
 458                 for($i = 0; $i < $iInstances; $i++)
 459                 {
 460                         $aDBInstances[$i] =& getDB(true);
 461                 }
 462
 463                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 464                 {
 465                         echo $sFile.': ';
 466                         $hFile = fopen($sFile, "r");
 467                         $sSQL = fgets($hFile, 100000);
 468                         $iLines = 0;
 469
 470                         while(true)
 471                         {
 472                                 for($i = 0; $i < $iInstances; $i++)
 473                                 {
 474                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 475                                         {
 476                                                 while(pg_get_result($aDBInstances[$i]->connection));
 477                                                 $sSQL = fgets($hFile, 100000);
 478                                                 if (!$sSQL) break 2;
 479                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 480                                                 $iLines++;
 481                                                 if ($iLines == 1000)
 482                                                 {
 483                                                         echo ".";
 484                                                         $iLines = 0;
 485                                                 }
 486                                         }
 487                                 }
 488                                 usleep(10);
 489                         }
 490
 491                         fclose($hFile);
 492
 493                         $bAnyBusy = true;
 494                         while($bAnyBusy)
 495                         {
 496                                 $bAnyBusy = false;
 497                                 for($i = 0; $i < $iInstances; $i++)
 498                                 {
 499                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 500                                 }
 501                                 usleep(10);
 502                         }
 503                         echo "\n";
 504                 }
 505
 506                 echo "Creating indexes\n";
 507                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 508         }
 509
 510         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 511         {
 512                 $bDidSomething = true;
 513                 $oDB =& getDB();
 514                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 515                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 516                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 517                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 518                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 519                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 520                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 521
 522                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 523                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 524                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 525                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 526         }
 527
 528         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 529         {
 530                 $bDidSomething = true;
 531                 $oDB =& getDB();
 532
 533                 if (!file_exists(CONST_Osmosis_Binary))
 534                 {
 535                         echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 536                         if (!$aCMDResult['all'])
 537                         {
 538                                 fail("osmosis not found in '".CONST_Osmosis_Binary."'");
 539                         }
 540                 }
 541                 else
 542                 {
 543                         if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 544                         {
 545                                 echo "settings/configuration.txt already exists\n";
 546                         }
 547                         else
 548                         {
 549                                 passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 550                                 // update osmosis configuration.txt with our settings
 551                                 passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
 552                                 passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
 553                         }
 554
 555                         // Find the last node in the DB
 556                         $iLastOSMID = $oDB->getOne("select max(id) from planet_osm_nodes");
 557
 558                         // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 559                         $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
 560                         $sLastNodeXML = file_get_contents($sLastNodeURL);
 561                         preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 562                         $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 563
 564                         // Search for the correct state file - uses file timestamps so need to sort by date descending
 565                         $sRepURL = CONST_Replication_Url."/";
 566                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 567                         // download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
 568                         // planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
 569                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 570                         $aPrevRepMatch = false;
 571                         foreach($aRepMatches as $aRepMatch)
 572                         {
 573                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 574                                 $aPrevRepMatch = $aRepMatch;
 575                         }
 576                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 577
 578                         $sRepURL .= $aRepMatch[1];
 579                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 580                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 581                         $aPrevRepMatch = false;
 582                         foreach($aRepMatches as $aRepMatch)
 583                         {
 584                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 585                                 $aPrevRepMatch = $aRepMatch;
 586                         }
 587                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 588
 589                         $sRepURL .= $aRepMatch[1];
 590                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 591                         preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 592                         $aPrevRepMatch = false;
 593                         foreach($aRepMatches as $aRepMatch)
 594                         {
 595                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 596                                 $aPrevRepMatch = $aRepMatch;
 597                         }
 598                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 599
 600                         $sRepURL .= $aRepMatch[1].'.state.txt';
 601                         echo "Getting state file: $sRepURL\n";
 602                         $sStateFile = file_get_contents($sRepURL);
 603                         if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 604                         file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 605                         echo "Updating DB status\n";
 606                         pg_query($oDB->connection, 'TRUNCATE import_status');
 607                         $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 608                         pg_query($oDB->connection, $sSQL);
 609                 }
 610         }
 611
 612         if ($aCMDResult['index'] || $aCMDResult['all'])
 613         {
 614                 $bDidSomething = true;
 615                 $sOutputFile = '';
 616                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 617                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 618                 passthruCheckReturn($sBaseCmd.' -R 4');
 619                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 620                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 621                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 622                 passthruCheckReturn($sBaseCmd.' -r 26');
 623         }
 624
 625         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 626         {
 627                 echo "Search indices\n";
 628                 $bDidSomething = true;
 629                 $oDB =& getDB();
 630                 $sSQL = 'select distinct partition from country_name';
 631                 $aPartitions = $oDB->getCol($sSQL);
 632                 if (PEAR::isError($aPartitions))
 633                 {
 634                         fail($aPartitions->getMessage());
 635                 }
 636                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 637
 638                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 639                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 640                 foreach($aMatches as $aMatch)
 641                 {
 642                         $sResult = '';
 643                         foreach($aPartitions as $sPartitionName)
 644                         {
 645                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 646                         }
 647                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 648                 }
 649
 650                 pgsqlRunScript($sTemplate);
 651         }
 652
 653         if (isset($aCMDResult['create-website']))
 654         {
 655                 $bDidSomething = true;
 656                 $sTargetDir = $aCMDResult['create-website'];
 657                 if (!is_dir($sTargetDir))
 658                 {
 659                         echo "You must create the website directory before calling this function.\n";
 660                         fail("Target directory does not exist.");
 661                 }
 662
 663                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 664                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 665                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 666                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 667                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 668                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 669                 @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php');
 670                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 671                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 672                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 673                 echo "Symlinks created\n";
 674
 675                 $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
 676                 if (!$sTestFile)
 677                 {
 678                         echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
 679                         echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
 680                 }
 681         }
 682
 683         if (!$bDidSomething)
 684         {
 685                 showUsage($aCMDOptions, true);
 686         }
 687
 688         function pgsqlRunScriptFile($sFilename)
 689         {
 690                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 691
 692                 // Convert database DSN to psql parameters
 693                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 694                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 695                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 696
 697                 $ahGzipPipes = null;
 698                 if (preg_match('/\\.gz$/', $sFilename))
 699                 {
 700                         $aDescriptors = array(
 701                                 0 => array('pipe', 'r'),
 702                                 1 => array('pipe', 'w'),
 703                                 2 => array('file', '/dev/null', 'a')
 704                         );
 705                         $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
 706                         if (!is_resource($hGzipProcess)) fail('unable to start zcat');
 707                         $aReadPipe = $ahGzipPipes[1];
 708                         fclose($ahGzipPipes[0]);
 709                 }
 710                 else
 711                 {
 712                         $sCMD .= ' -f '.$sFilename;
 713                         $aReadPipe = array('pipe', 'r');
 714                 }
 715
 716                 $aDescriptors = array(
 717                         0 => $aReadPipe,
 718                         1 => array('pipe', 'w'),
 719                         2 => array('file', '/dev/null', 'a')
 720                 );
 721                 $ahPipes = null;
 722                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 723                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 724
 725
 726                 // TODO: error checking
 727                 while(!feof($ahPipes[1]))
 728                 {
 729                         echo fread($ahPipes[1], 4096);
 730                 }
 731                 fclose($ahPipes[1]);
 732
 733                 $iReturn = proc_close($hProcess);
 734                 if ($iReturn > 0)
 735                 {
 736                         fail("pgsql returned with error code ($iReturn)");
 737                 }
 738                 if ($ahGzipPipes)
 739                 {
 740                         fclose($ahGzipPipes[1]);
 741                         proc_close($hGzipProcess);
 742                 }
 743
 744         }
 745
 746         function pgsqlRunScript($sScript)
 747         {
 748                 // Convert database DSN to psql parameters
 749                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 750                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 751                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 752                 $sCMD .= ' -v ON_ERROR_STOP=1';
 753                 $aDescriptors = array(
 754                         0 => array('pipe', 'r'),
 755                         1 => STDOUT,
 756                         2 => STDERR
 757                 );
 758                 $ahPipes = null;
 759                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 760                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 761
 762                 while(strlen($sScript))
 763                 {
 764                         $written = fwrite($ahPipes[0], $sScript);
 765                         if ($written <= 0) break;
 766                         $sScript = substr($sScript, $written);
 767                 }
 768                 fclose($ahPipes[0]);
 769                 $iReturn = proc_close($hProcess);
 770                 if ($iReturn > 0)
 771                 {
 772                         fail("pgsql returned with error code ($iReturn)");
 773                 }
 774         }
 775
 776         function pgsqlRunRestoreData($sDumpFile)
 777         {
 778                 // Convert database DSN to psql parameters
 779                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 780                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 781                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 782
 783                 $aDescriptors = array(
 784                         0 => array('pipe', 'r'),
 785                         1 => array('pipe', 'w'),
 786                         2 => array('file', '/dev/null', 'a')
 787                 );
 788                 $ahPipes = null;
 789                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 790                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 791
 792                 fclose($ahPipes[0]);
 793
 794                 // TODO: error checking
 795                 while(!feof($ahPipes[1]))
 796                 {
 797                         echo fread($ahPipes[1], 4096);
 798                 }
 799                 fclose($ahPipes[1]);
 800
 801                 $iReturn = proc_close($hProcess);
 802                 if ($iReturn > 0)
 803                 {
 804                         fail("pgsql returned with error code ($iReturn)");
 805                 }
 806         }
 807
 808         function pgsqlRunDropAndRestore($sDumpFile)
 809         {
 810                 // Convert database DSN to psql parameters
 811                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 812                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 813                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 814
 815                 $aDescriptors = array(
 816                         0 => array('pipe', 'r'),
 817                         1 => array('pipe', 'w'),
 818                         2 => array('file', '/dev/null', 'a')
 819                 );
 820                 $ahPipes = null;
 821                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 822                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 823
 824                 fclose($ahPipes[0]);
 825
 826                 // TODO: error checking
 827                 while(!feof($ahPipes[1]))
 828                 {
 829                         echo fread($ahPipes[1], 4096);
 830                 }
 831                 fclose($ahPipes[1]);
 832
 833                 $iReturn = proc_close($hProcess);
 834                 if ($iReturn > 0)
 835                 {
 836                         fail("pgsql returned with error code ($iReturn)");
 837                 }
 838         }
 839
 840         function passthruCheckReturn($cmd)
 841         {
 842                 $result = -1;
 843                 passthru($cmd, $result);
 844                 if ($result != 0) fail('Error executing external command: '.$cmd);
 845         }