utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
  28                 array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
  29                 array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
  30                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  31                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  32                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  33                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  34                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  35                 array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
  36                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  38                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  39                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  40                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  41                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  42         );
  43         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  44
  45         $bDidSomething = false;
  46
  47         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  48         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  49         {
  50                 if (!isset($aCMDResult['osm-file']))
  51                 {
  52                         fail('missing --osm-file for data import');
  53                 }
  54
  55                 if (!file_exists($aCMDResult['osm-file']))
  56                 {
  57                         fail('the path supplied to --osm-file does not exist');
  58                 }
  59
  60                 if (!is_readable($aCMDResult['osm-file']))
  61                 {
  62                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  63                 }
  64         }
  65
  66
  67         // This is a pretty hard core default - the number of processors in the box - 1
  68         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  69         if ($iInstances < 1)
  70         {
  71                 $iInstances = 1;
  72                 echo "WARNING: resetting threads to $iInstances\n";
  73         }
  74         if ($iInstances > getProcessorCount())
  75         {
  76                 $iInstances = getProcessorCount();
  77                 echo "WARNING: resetting threads to $iInstances\n";
  78         }
  79
  80         // Assume we can steal all the cache memory in the box (unless told otherwise)
  81         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  82         if ($iCacheMemory > getTotalMemoryMB())
  83         {
  84                 $iCacheMemory = getCacheMemoryMB();
  85                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  86         }
  87
  88         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  89         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  90
  91         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  92         {
  93                 echo "Create DB\n";
  94                 $bDidSomething = true;
  95                 $oDB =& DB::connect(CONST_Database_DSN, false);
  96                 if (!PEAR::isError($oDB))
  97                 {
  98                         fail('database already exists ('.CONST_Database_DSN.')');
  99                 }
 100                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 101         }
 102
 103         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 104         {
 105                 echo "Setup DB\n";
 106                 $bDidSomething = true;
 107                 // TODO: path detection, detection memory, etc.
 108
 109                 $oDB =& getDB();
 110
 111                 $sVersionString = $oDB->getOne('select version()');
 112                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 113                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 114                 {
 115                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 116                         exit;
 117                 }
 118
 119                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 124                 } else {
 125                         pgsqlRunScript('CREATE EXTENSION hstore');
 126                 }
 127
 128                 $fPostgisVersion = (float) CONST_Postgis_Version;
 129                 if ($fPostgisVersion < 2.0) {
 130                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 131                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 132                 } else {
 133                         pgsqlRunScript('CREATE EXTENSION postgis');
 134                 }
 135                 if ($fPostgisVersion < 2.1) {
 136                         // Function was renamed in 2.1 and throws an annoying deprecation warning
 137                         pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
 138                 }
 139                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 140                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 141                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 142                 {
 143                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 144                         exit;
 145                 }
 146
 147                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 148                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 149                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 150                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 151                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 152                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 153                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 154
 155                 if ($aCMDResult['no-partitions'])
 156                 {
 157                         pgsqlRunScript('update country_name set partition = 0');
 158                 }
 159         }
 160
 161         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 162         {
 163                 echo "Import\n";
 164                 $bDidSomething = true;
 165
 166                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 167                 if (!file_exists($osm2pgsql))
 168                 {
 169                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 170                         fail("osm2pgsql not found in '$osm2pgsql'");
 171                 }
 172
 173                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 174                 {
 175                         $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 176                 }
 177                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 178                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 179                 $osm2pgsql .= ' -C 18000';
 180                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 181                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 182                 passthruCheckReturn($osm2pgsql);
 183
 184                 $oDB =& getDB();
 185                 $x = $oDB->getRow('select * from place limit 1');
 186                 if (PEAR::isError($x)) {
 187                         fail($x->getMessage());
 188                 }
 189                 if (!$x) fail('No Data');
 190         }
 191
 192         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 193         {
 194                 echo "Functions\n";
 195                 $bDidSomething = true;
 196                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 197                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 198                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 199                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 200                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 201                 if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
 202                 pgsqlRunScript($sTemplate);
 203         }
 204
 205         if ($aCMDResult['create-minimal-tables'])
 206         {
 207                 echo "Minimal Tables\n";
 208                 $bDidSomething = true;
 209                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 210
 211                 $sScript = '';
 212
 213                 // Backstop the import process - easliest possible import id
 214                 $sScript .= "insert into import_npi_log values (18022);\n";
 215
 216                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 217                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 218
 219                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 220                 {
 221                         list($sClass, $sType) = explode(' ', trim($sLine));
 222                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 223                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 224
 225                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 226                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 227
 228                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 229                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 230                 }
 231                 fclose($hFile);
 232                 pgsqlRunScript($sScript);
 233         }
 234
 235         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 236         {
 237                 echo "Tables\n";
 238                 $bDidSomething = true;
 239                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 240
 241                 // re-run the functions
 242                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 243                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 244                 pgsqlRunScript($sTemplate);
 245         }
 246
 247         if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 248         {
 249                 echo "Partition Tables\n";
 250                 $bDidSomething = true;
 251                 $oDB =& getDB();
 252                 $sSQL = 'select distinct partition from country_name';
 253                 $aPartitions = $oDB->getCol($sSQL);
 254                 if (PEAR::isError($aPartitions))
 255                 {
 256                         fail($aPartitions->getMessage());
 257                 }
 258                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 259
 260                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
 261                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 262                 foreach($aMatches as $aMatch)
 263                 {
 264                         $sResult = '';
 265                         foreach($aPartitions as $sPartitionName)
 266                         {
 267                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 268                         }
 269                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 270                 }
 271
 272                 pgsqlRunScript($sTemplate);
 273         }
 274
 275
 276         if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
 277         {
 278                 echo "Partition Functions\n";
 279                 $bDidSomething = true;
 280                 $oDB =& getDB();
 281                 $sSQL = 'select distinct partition from country_name';
 282                 $aPartitions = $oDB->getCol($sSQL);
 283                 if (PEAR::isError($aPartitions))
 284                 {
 285                         fail($aPartitions->getMessage());
 286                 }
 287                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 288
 289                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 290                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 291                 foreach($aMatches as $aMatch)
 292                 {
 293                         $sResult = '';
 294                         foreach($aPartitions as $sPartitionName)
 295                         {
 296                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 297                         }
 298                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 299                 }
 300
 301                 pgsqlRunScript($sTemplate);
 302         }
 303
 304         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 305         {
 306                 $bDidSomething = true;
 307                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 308                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 309                 if (file_exists($sWikiArticlesFile))
 310                 {
 311                         echo "Importing wikipedia articles...";
 312                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 313                         echo "...done\n";
 314                 }
 315                 else
 316                 {
 317                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 318                 }
 319                 if (file_exists($sWikiRedirectsFile))
 320                 {
 321                         echo "Importing wikipedia redirects...";
 322                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 323                         echo "...done\n";
 324                 }
 325                 else
 326                 {
 327                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 328                 }
 329         }
 330
 331
 332         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 333         {
 334                 echo "Drop old Data\n";
 335                 $bDidSomething = true;
 336
 337                 $oDB =& getDB();
 338                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 339                 echo '.';
 340                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 341                 echo '.';
 342                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 343                 echo '.';
 344                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 345                 echo '.';
 346                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 347                 echo '.';
 348                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 349                 echo '.';
 350                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 351                 echo '.';
 352                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 353                 echo '.';
 354                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 355                 echo '.';
 356
 357                 $sSQL = 'select distinct partition from country_name';
 358                 $aPartitions = $oDB->getCol($sSQL);
 359                 if (PEAR::isError($aPartitions))
 360                 {
 361                         fail($aPartitions->getMessage());
 362                 }
 363                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 364                 foreach($aPartitions as $sPartition)
 365                 {
 366                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 367                         echo '.';
 368                 }
 369
 370                 // used by getorcreate_word_id to ignore frequent partial words
 371                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 372                 echo ".\n";
 373
 374                 // pre-create the word list
 375                 if (!$aCMDResult['disable-token-precalc'])
 376                 {
 377                         echo "Loading word list\n";
 378                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 379                 }
 380
 381                 echo "Load Data\n";
 382                 $aDBInstances = array();
 383                 for($i = 0; $i < $iInstances; $i++)
 384                 {
 385                         $aDBInstances[$i] =& getDB(true);
 386                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 387                         $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 388                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 389                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 390                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 391                 }
 392                 $bAnyBusy = true;
 393                 while($bAnyBusy)
 394                 {
 395                         $bAnyBusy = false;
 396                         for($i = 0; $i < $iInstances; $i++)
 397                         {
 398                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 399                         }
 400                         sleep(1);
 401                         echo '.';
 402                 }
 403                 echo "\n";
 404                 echo "Reanalysing database...\n";
 405                 pgsqlRunScript('ANALYSE');
 406         }
 407
 408         if ($aCMDResult['create-roads'])
 409         {
 410                 $bDidSomething = true;
 411
 412                 $oDB =& getDB();
 413                 $aDBInstances = array();
 414                 for($i = 0; $i < $iInstances; $i++)
 415                 {
 416                         $aDBInstances[$i] =& getDB(true);
 417                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 418                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 419                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 420                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 421                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 422                 }
 423                 $bAnyBusy = true;
 424                 while($bAnyBusy)
 425                 {
 426                         $bAnyBusy = false;
 427                         for($i = 0; $i < $iInstances; $i++)
 428                         {
 429                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 430                         }
 431                         sleep(1);
 432                         echo '.';
 433                 }
 434                 echo "\n";
 435         }
 436
 437         if ($aCMDResult['import-tiger-data'])
 438         {
 439                 $bDidSomething = true;
 440
 441                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 442
 443                 $aDBInstances = array();
 444                 for($i = 0; $i < $iInstances; $i++)
 445                 {
 446                         $aDBInstances[$i] =& getDB(true);
 447                 }
 448
 449                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 450                 {
 451                         echo $sFile.': ';
 452                         $hFile = fopen($sFile, "r");
 453                         $sSQL = fgets($hFile, 100000);
 454                         $iLines = 0;
 455
 456                         while(true)
 457                         {
 458                                 for($i = 0; $i < $iInstances; $i++)
 459                                 {
 460                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 461                                         {
 462                                                 while(pg_get_result($aDBInstances[$i]->connection));
 463                                                 $sSQL = fgets($hFile, 100000);
 464                                                 if (!$sSQL) break 2;
 465                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 466                                                 $iLines++;
 467                                                 if ($iLines == 1000)
 468                                                 {
 469                                                         echo ".";
 470                                                         $iLines = 0;
 471                                                 }
 472                                         }
 473                                 }
 474                                 usleep(10);
 475                         }
 476
 477                         fclose($hFile);
 478
 479                         $bAnyBusy = true;
 480                         while($bAnyBusy)
 481                         {
 482                                 $bAnyBusy = false;
 483                                 for($i = 0; $i < $iInstances; $i++)
 484                                 {
 485                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 486                                 }
 487                                 usleep(10);
 488                         }
 489                         echo "\n";
 490                 }
 491
 492                 echo "Creating indexes\n";
 493                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 494         }
 495
 496         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 497         {
 498                 $bDidSomething = true;
 499                 $oDB =& getDB();
 500                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 501                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 502                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 503                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 504                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 505                 $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";
 506                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 507
 508                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 509                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 510                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 511                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 512         }
 513
 514         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 515         {
 516                 $bDidSomething = true;
 517                 $oDB =& getDB();
 518
 519                 if (!file_exists(CONST_Osmosis_Binary))
 520                 {
 521                         echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 522                         if (!$aCMDResult['all'])
 523                         {
 524                                 fail("osmosis not found in '".CONST_Osmosis_Binary."'");
 525                         }
 526                 }
 527                 else
 528                 {
 529                         if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 530                         {
 531                                 echo "settings/configuration.txt already exists\n";
 532                         }
 533                         else
 534                         {
 535                                 passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 536                                 // update osmosis configuration.txt with our settings
 537                                 passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
 538                                 passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
 539                         }
 540
 541                         // Find the last node in the DB
 542                         $iLastOSMID = $oDB->getOne("select max(id) from planet_osm_nodes");
 543
 544                         // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 545                         $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
 546                         $sLastNodeXML = file_get_contents($sLastNodeURL);
 547                         preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 548                         $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 549
 550                         // Search for the correct state file - uses file timestamps so need to sort by date descending
 551                         $sRepURL = CONST_Replication_Url."/";
 552                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 553                         // download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
 554                         // planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
 555                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 556                         $aPrevRepMatch = false;
 557                         foreach($aRepMatches as $aRepMatch)
 558                         {
 559                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 560                                 $aPrevRepMatch = $aRepMatch;
 561                         }
 562                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 563
 564                         $sRepURL .= $aRepMatch[1];
 565                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 566                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 567                         $aPrevRepMatch = false;
 568                         foreach($aRepMatches as $aRepMatch)
 569                         {
 570                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 571                                 $aPrevRepMatch = $aRepMatch;
 572                         }
 573                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 574
 575                         $sRepURL .= $aRepMatch[1];
 576                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 577                         preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 578                         $aPrevRepMatch = false;
 579                         foreach($aRepMatches as $aRepMatch)
 580                         {
 581                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 582                                 $aPrevRepMatch = $aRepMatch;
 583                         }
 584                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 585
 586                         $sRepURL .= $aRepMatch[1].'.state.txt';
 587                         echo "Getting state file: $sRepURL\n";
 588                         $sStateFile = file_get_contents($sRepURL);
 589                         if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 590                         file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 591                         echo "Updating DB status\n";
 592                         pg_query($oDB->connection, 'TRUNCATE import_status');
 593                         $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 594                         pg_query($oDB->connection, $sSQL);
 595                 }
 596         }
 597
 598         if ($aCMDResult['index'] || $aCMDResult['all'])
 599         {
 600                 $bDidSomething = true;
 601                 $sOutputFile = '';
 602                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 603                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 604                 passthruCheckReturn($sBaseCmd.' -R 4');
 605                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 606                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 607                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 608                 passthruCheckReturn($sBaseCmd.' -r 26');
 609         }
 610
 611         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 612         {
 613                 echo "Search indices\n";
 614                 $bDidSomething = true;
 615                 $oDB =& getDB();
 616                 $sSQL = 'select distinct partition from country_name';
 617                 $aPartitions = $oDB->getCol($sSQL);
 618                 if (PEAR::isError($aPartitions))
 619                 {
 620                         fail($aPartitions->getMessage());
 621                 }
 622                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 623
 624                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 625                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 626                 foreach($aMatches as $aMatch)
 627                 {
 628                         $sResult = '';
 629                         foreach($aPartitions as $sPartitionName)
 630                         {
 631                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 632                         }
 633                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 634                 }
 635
 636                 pgsqlRunScript($sTemplate);
 637         }
 638
 639         if (isset($aCMDResult['create-website']))
 640         {
 641                 $bDidSomething = true;
 642                 $sTargetDir = $aCMDResult['create-website'];
 643                 if (!is_dir($sTargetDir))
 644                 {
 645                         echo "You must create the website directory before calling this function.\n";
 646                         fail("Target directory does not exist.");
 647                 }
 648
 649                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 650                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 651                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 652                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 653                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 654                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 655                 @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php');
 656                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 657                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 658                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 659                 echo "Symlinks created\n";
 660
 661                 $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
 662                 if (!$sTestFile)
 663                 {
 664                         echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
 665                         echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
 666                 }
 667         }
 668
 669         if (!$bDidSomething)
 670         {
 671                 showUsage($aCMDOptions, true);
 672         }
 673
 674         function pgsqlRunScriptFile($sFilename)
 675         {
 676                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 677
 678                 // Convert database DSN to psql parameters
 679                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 680                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 681                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 682
 683                 $aDescriptors = array(
 684                         0 => array('pipe', 'r'),
 685                         1 => array('pipe', 'w'),
 686                         2 => array('file', '/dev/null', 'a')
 687                 );
 688                 $ahPipes = null;
 689                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 690                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 691
 692                 fclose($ahPipes[0]);
 693
 694                 // TODO: error checking
 695                 while(!feof($ahPipes[1]))
 696                 {
 697                         echo fread($ahPipes[1], 4096);
 698                 }
 699                 fclose($ahPipes[1]);
 700
 701                 proc_close($hProcess);
 702         }
 703
 704         function pgsqlRunScript($sScript)
 705         {
 706                 // Convert database DSN to psql parameters
 707                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 708                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 709                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 710                 $aDescriptors = array(
 711                         0 => array('pipe', 'r'),
 712                         1 => STDOUT,
 713                         2 => STDERR
 714                 );
 715                 $ahPipes = null;
 716                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 717                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 718
 719                 while(strlen($sScript))
 720                 {
 721                         $written = fwrite($ahPipes[0], $sScript);
 722                         $sScript = substr($sScript, $written);
 723                 }
 724                 fclose($ahPipes[0]);
 725                 proc_close($hProcess);
 726         }
 727
 728         function pgsqlRunRestoreData($sDumpFile)
 729         {
 730                 // Convert database DSN to psql parameters
 731                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 732                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 733                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 734
 735                 $aDescriptors = array(
 736                         0 => array('pipe', 'r'),
 737                         1 => array('pipe', 'w'),
 738                         2 => array('file', '/dev/null', 'a')
 739                 );
 740                 $ahPipes = null;
 741                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 742                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 743
 744                 fclose($ahPipes[0]);
 745
 746                 // TODO: error checking
 747                 while(!feof($ahPipes[1]))
 748                 {
 749                         echo fread($ahPipes[1], 4096);
 750                 }
 751                 fclose($ahPipes[1]);
 752
 753                 proc_close($hProcess);
 754         }
 755
 756         function pgsqlRunDropAndRestore($sDumpFile)
 757         {
 758                 // Convert database DSN to psql parameters
 759                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 760                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 761                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 762
 763                 $aDescriptors = array(
 764                         0 => array('pipe', 'r'),
 765                         1 => array('pipe', 'w'),
 766                         2 => array('file', '/dev/null', 'a')
 767                 );
 768                 $ahPipes = null;
 769                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 770                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 771
 772                 fclose($ahPipes[0]);
 773
 774                 // TODO: error checking
 775                 while(!feof($ahPipes[1]))
 776                 {
 777                         echo fread($ahPipes[1], 4096);
 778                 }
 779                 fclose($ahPipes[1]);
 780
 781                 proc_close($hProcess);
 782         }
 783
 784         function passthruCheckReturn($cmd)
 785         {
 786                 $result = -1;
 787                 passthru($cmd, $result);
 788                 if ($result != 0) fail('Error executing external command: '.$cmd);
 789         }