utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
  28                 array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
  29                 array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
  30                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  31                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  32                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  33                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  34                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  35                 array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
  36                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  38                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  39                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  40                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  41                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  42         );
  43         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  44
  45         $bDidSomething = false;
  46
  47         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  48         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  49         {
  50                 if (!isset($aCMDResult['osm-file']))
  51                 {
  52                         fail('missing --osm-file for data import');
  53                 }
  54
  55                 if (!file_exists($aCMDResult['osm-file']))
  56                 {
  57                         fail('the path supplied to --osm-file does not exist');
  58                 }
  59
  60                 if (!is_readable($aCMDResult['osm-file']))
  61                 {
  62                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  63                 }
  64         }
  65
  66
  67         // This is a pretty hard core default - the number of processors in the box - 1
  68         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  69         if ($iInstances < 1)
  70         {
  71                 $iInstances = 1;
  72                 echo "WARNING: resetting threads to $iInstances\n";
  73         }
  74         if ($iInstances > getProcessorCount())
  75         {
  76                 $iInstances = getProcessorCount();
  77                 echo "WARNING: resetting threads to $iInstances\n";
  78         }
  79
  80         // Assume we can steal all the cache memory in the box (unless told otherwise)
  81         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  82         if ($iCacheMemory > getTotalMemoryMB())
  83         {
  84                 $iCacheMemory = getCacheMemoryMB();
  85                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  86         }
  87
  88         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  89         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  90
  91         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  92         {
  93                 echo "Create DB\n";
  94                 $bDidSomething = true;
  95                 $oDB =& DB::connect(CONST_Database_DSN, false);
  96                 if (!PEAR::isError($oDB))
  97                 {
  98                         fail('database already exists ('.CONST_Database_DSN.')');
  99                 }
 100                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 101         }
 102
 103         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 104         {
 105                 echo "Setup DB\n";
 106                 $bDidSomething = true;
 107                 // TODO: path detection, detection memory, etc.
 108
 109                 $oDB =& getDB();
 110
 111                 $sVersionString = $oDB->getOne('select version()');
 112                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 113                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 114                 {
 115                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 116                         exit;
 117                 }
 118
 119                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 124                 } else {
 125                         pgsqlRunScript('CREATE EXTENSION hstore');
 126                 }
 127
 128                 $fPostgisVersion = (float) CONST_Postgis_Version;
 129                 if ($fPostgisVersion < 2.0) {
 130                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 131                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 132                 } else {
 133                         pgsqlRunScript('CREATE EXTENSION postgis');
 134                 }
 135                 if ($fPostgisVersion < 2.1) {
 136                         // Function was renamed in 2.1 and throws an annoying deprecation warning
 137                         pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
 138                 }
 139                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 140                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 141                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 142                 {
 143                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 144                         exit;
 145                 }
 146
 147                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 148                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 149                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 150                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 151                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 152                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 153                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 154
 155                 if ($aCMDResult['no-partitions'])
 156                 {
 157                         pgsqlRunScript('update country_name set partition = 0');
 158                 }
 159         }
 160
 161         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 162         {
 163                 echo "Import\n";
 164                 $bDidSomething = true;
 165
 166                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 167                 if (!file_exists($osm2pgsql))
 168                 {
 169                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 170                         fail("osm2pgsql not found in '$osm2pgsql'");
 171                 }
 172
 173                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 174                 {
 175                         $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 176                 }
 177                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 178                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 179                 $osm2pgsql .= ' -C 18000';
 180                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 181                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 182                 passthruCheckReturn($osm2pgsql);
 183
 184                 $oDB =& getDB();
 185                 $x = $oDB->getRow('select * from place limit 1');
 186                 if (PEAR::isError($x)) {
 187                         fail($x->getMessage());
 188                 }
 189                 if (!$x) fail('No Data');
 190         }
 191
 192         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 193         {
 194                 echo "Functions\n";
 195                 $bDidSomething = true;
 196                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 197                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 198                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 199                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 200                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 201                 pgsqlRunScript($sTemplate);
 202         }
 203
 204         if ($aCMDResult['create-minimal-tables'])
 205         {
 206                 echo "Minimal Tables\n";
 207                 $bDidSomething = true;
 208                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 209
 210                 $sScript = '';
 211
 212                 // Backstop the import process - easliest possible import id
 213                 $sScript .= "insert into import_npi_log values (18022);\n";
 214
 215                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 216                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 217
 218                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 219                 {
 220                         list($sClass, $sType) = explode(' ', trim($sLine));
 221                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 222                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 223
 224                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 225                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 226
 227                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 228                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 229                 }
 230                 fclose($hFile);
 231                 pgsqlRunScript($sScript);
 232         }
 233
 234         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 235         {
 236                 echo "Tables\n";
 237                 $bDidSomething = true;
 238                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 239
 240                 // re-run the functions
 241                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 242                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 243                 pgsqlRunScript($sTemplate);
 244         }
 245
 246         if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 247         {
 248                 echo "Partition Tables\n";
 249                 $bDidSomething = true;
 250                 $oDB =& getDB();
 251                 $sSQL = 'select distinct partition from country_name';
 252                 $aPartitions = $oDB->getCol($sSQL);
 253                 if (PEAR::isError($aPartitions))
 254                 {
 255                         fail($aPartitions->getMessage());
 256                 }
 257                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 258
 259                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
 260                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 261                 foreach($aMatches as $aMatch)
 262                 {
 263                         $sResult = '';
 264                         foreach($aPartitions as $sPartitionName)
 265                         {
 266                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 267                         }
 268                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 269                 }
 270
 271                 pgsqlRunScript($sTemplate);
 272         }
 273
 274
 275         if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
 276         {
 277                 echo "Partition Functions\n";
 278                 $bDidSomething = true;
 279                 $oDB =& getDB();
 280                 $sSQL = 'select distinct partition from country_name';
 281                 $aPartitions = $oDB->getCol($sSQL);
 282                 if (PEAR::isError($aPartitions))
 283                 {
 284                         fail($aPartitions->getMessage());
 285                 }
 286                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 287
 288                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 289                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 290                 foreach($aMatches as $aMatch)
 291                 {
 292                         $sResult = '';
 293                         foreach($aPartitions as $sPartitionName)
 294                         {
 295                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 296                         }
 297                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 298                 }
 299
 300                 pgsqlRunScript($sTemplate);
 301         }
 302
 303         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 304         {
 305                 $bDidSomething = true;
 306                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 307                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 308                 if (file_exists($sWikiArticlesFile))
 309                 {
 310                         echo "Importing wikipedia articles...";
 311                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 312                         echo "...done\n";
 313                 }
 314                 else
 315                 {
 316                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 317                 }
 318                 if (file_exists($sWikiRedirectsFile))
 319                 {
 320                         echo "Importing wikipedia redirects...";
 321                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 322                         echo "...done\n";
 323                 }
 324                 else
 325                 {
 326                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 327                 }
 328         }
 329
 330
 331         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 332         {
 333                 echo "Drop old Data\n";
 334                 $bDidSomething = true;
 335
 336                 $oDB =& getDB();
 337                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 338                 echo '.';
 339                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 340                 echo '.';
 341                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 342                 echo '.';
 343                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 344                 echo '.';
 345                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 346                 echo '.';
 347                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 348                 echo '.';
 349                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 350                 echo '.';
 351                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 352                 echo '.';
 353                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 354                 echo '.';
 355
 356                 $sSQL = 'select distinct partition from country_name';
 357                 $aPartitions = $oDB->getCol($sSQL);
 358                 if (PEAR::isError($aPartitions))
 359                 {
 360                         fail($aPartitions->getMessage());
 361                 }
 362                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 363                 foreach($aPartitions as $sPartition)
 364                 {
 365                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 366                         echo '.';
 367                 }
 368
 369                 // used by getorcreate_word_id to ignore frequent partial words
 370                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 371                 echo ".\n";
 372
 373                 // pre-create the word list
 374                 if (!$aCMDResult['disable-token-precalc'])
 375                 {
 376                         echo "Loading word list\n";
 377                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 378                 }
 379
 380                 echo "Load Data\n";
 381                 $aDBInstances = array();
 382                 for($i = 0; $i < $iInstances; $i++)
 383                 {
 384                         $aDBInstances[$i] =& getDB(true);
 385                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 386                         $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 387                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 388                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 389                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 390                 }
 391                 $bAnyBusy = true;
 392                 while($bAnyBusy)
 393                 {
 394                         $bAnyBusy = false;
 395                         for($i = 0; $i < $iInstances; $i++)
 396                         {
 397                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 398                         }
 399                         sleep(1);
 400                         echo '.';
 401                 }
 402                 echo "\n";
 403                 echo "Reanalysing database...\n";
 404                 pgsqlRunScript('ANALYSE');
 405         }
 406
 407         if ($aCMDResult['create-roads'])
 408         {
 409                 $bDidSomething = true;
 410
 411                 $oDB =& getDB();
 412                 $aDBInstances = array();
 413                 for($i = 0; $i < $iInstances; $i++)
 414                 {
 415                         $aDBInstances[$i] =& getDB(true);
 416                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 417                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 418                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 419                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 420                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 421                 }
 422                 $bAnyBusy = true;
 423                 while($bAnyBusy)
 424                 {
 425                         $bAnyBusy = false;
 426                         for($i = 0; $i < $iInstances; $i++)
 427                         {
 428                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 429                         }
 430                         sleep(1);
 431                         echo '.';
 432                 }
 433                 echo "\n";
 434         }
 435
 436         if ($aCMDResult['import-tiger-data'])
 437         {
 438                 $bDidSomething = true;
 439
 440                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 441
 442                 $aDBInstances = array();
 443                 for($i = 0; $i < $iInstances; $i++)
 444                 {
 445                         $aDBInstances[$i] =& getDB(true);
 446                 }
 447
 448                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 449                 {
 450                         echo $sFile.': ';
 451                         $hFile = fopen($sFile, "r");
 452                         $sSQL = fgets($hFile, 100000);
 453                         $iLines = 0;
 454
 455                         while(true)
 456                         {
 457                                 for($i = 0; $i < $iInstances; $i++)
 458                                 {
 459                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 460                                         {
 461                                                 while(pg_get_result($aDBInstances[$i]->connection));
 462                                                 $sSQL = fgets($hFile, 100000);
 463                                                 if (!$sSQL) break 2;
 464                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 465                                                 $iLines++;
 466                                                 if ($iLines == 1000)
 467                                                 {
 468                                                         echo ".";
 469                                                         $iLines = 0;
 470                                                 }
 471                                         }
 472                                 }
 473                                 usleep(10);
 474                         }
 475
 476                         fclose($hFile);
 477
 478                         $bAnyBusy = true;
 479                         while($bAnyBusy)
 480                         {
 481                                 $bAnyBusy = false;
 482                                 for($i = 0; $i < $iInstances; $i++)
 483                                 {
 484                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 485                                 }
 486                                 usleep(10);
 487                         }
 488                         echo "\n";
 489                 }
 490
 491                 echo "Creating indexes\n";
 492                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 493         }
 494
 495         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 496         {
 497                 $bDidSomething = true;
 498                 $oDB =& getDB();
 499                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 500                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 501                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 502                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 503                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 504                 $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";
 505                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 506
 507                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 508                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 509                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 510                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 511         }
 512
 513         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 514         {
 515                 $bDidSomething = true;
 516                 $oDB =& getDB();
 517
 518                 if (!file_exists(CONST_Osmosis_Binary))
 519                 {
 520                         echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 521                         if (!$aCMDResult['all'])
 522                         {
 523                                 fail("osmosis not found in '".CONST_Osmosis_Binary."'");
 524                         }
 525                 }
 526                 else
 527                 {
 528                         if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 529                         {
 530                                 echo "settings/configuration.txt already exists\n";
 531                         }
 532                         else
 533                         {
 534                                 passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 535                                 // update osmosis configuration.txt with our settings
 536                                 passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
 537                                 passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
 538                         }
 539
 540                         // Find the last node in the DB
 541                         $iLastOSMID = $oDB->getOne("select max(id) from planet_osm_nodes");
 542
 543                         // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 544                         $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
 545                         $sLastNodeXML = file_get_contents($sLastNodeURL);
 546                         preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 547                         $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 548
 549                         // Search for the correct state file - uses file timestamps so need to sort by date descending
 550                         $sRepURL = CONST_Replication_Url."/";
 551                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 552                         // download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
 553                         // planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
 554                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 555                         $aPrevRepMatch = false;
 556                         foreach($aRepMatches as $aRepMatch)
 557                         {
 558                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 559                                 $aPrevRepMatch = $aRepMatch;
 560                         }
 561                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 562
 563                         $sRepURL .= $aRepMatch[1];
 564                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 565                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 566                         $aPrevRepMatch = false;
 567                         foreach($aRepMatches as $aRepMatch)
 568                         {
 569                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 570                                 $aPrevRepMatch = $aRepMatch;
 571                         }
 572                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 573
 574                         $sRepURL .= $aRepMatch[1];
 575                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 576                         preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 577                         $aPrevRepMatch = false;
 578                         foreach($aRepMatches as $aRepMatch)
 579                         {
 580                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 581                                 $aPrevRepMatch = $aRepMatch;
 582                         }
 583                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 584
 585                         $sRepURL .= $aRepMatch[1].'.state.txt';
 586                         echo "Getting state file: $sRepURL\n";
 587                         $sStateFile = file_get_contents($sRepURL);
 588                         if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 589                         file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 590                         echo "Updating DB status\n";
 591                         pg_query($oDB->connection, 'TRUNCATE import_status');
 592                         $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 593                         pg_query($oDB->connection, $sSQL);
 594                 }
 595         }
 596
 597         if ($aCMDResult['index'] || $aCMDResult['all'])
 598         {
 599                 $bDidSomething = true;
 600                 $sOutputFile = '';
 601                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 602                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 603                 passthruCheckReturn($sBaseCmd.' -R 4');
 604                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 605                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 606                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 607                 passthruCheckReturn($sBaseCmd.' -r 26');
 608         }
 609
 610         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 611         {
 612                 echo "Search indices\n";
 613                 $bDidSomething = true;
 614                 $oDB =& getDB();
 615                 $sSQL = 'select distinct partition from country_name';
 616                 $aPartitions = $oDB->getCol($sSQL);
 617                 if (PEAR::isError($aPartitions))
 618                 {
 619                         fail($aPartitions->getMessage());
 620                 }
 621                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 622
 623                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 624                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 625                 foreach($aMatches as $aMatch)
 626                 {
 627                         $sResult = '';
 628                         foreach($aPartitions as $sPartitionName)
 629                         {
 630                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 631                         }
 632                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 633                 }
 634
 635                 pgsqlRunScript($sTemplate);
 636         }
 637
 638         if (isset($aCMDResult['create-website']))
 639         {
 640                 $bDidSomething = true;
 641                 $sTargetDir = $aCMDResult['create-website'];
 642                 if (!is_dir($sTargetDir))
 643                 {
 644                         echo "You must create the website directory before calling this function.\n";
 645                         fail("Target directory does not exist.");
 646                 }
 647
 648                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 649                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 650                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 651                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 652                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 653                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 654                 @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php');
 655                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 656                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 657                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 658                 echo "Symlinks created\n";
 659
 660                 $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
 661                 if (!$sTestFile)
 662                 {
 663                         echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
 664                         echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
 665                 }
 666         }
 667
 668         if (!$bDidSomething)
 669         {
 670                 showUsage($aCMDOptions, true);
 671         }
 672
 673         function pgsqlRunScriptFile($sFilename)
 674         {
 675                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 676
 677                 // Convert database DSN to psql parameters
 678                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 679                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 680                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 681
 682                 $aDescriptors = array(
 683                         0 => array('pipe', 'r'),
 684                         1 => array('pipe', 'w'),
 685                         2 => array('file', '/dev/null', 'a')
 686                 );
 687                 $ahPipes = null;
 688                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 689                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 690
 691                 fclose($ahPipes[0]);
 692
 693                 // TODO: error checking
 694                 while(!feof($ahPipes[1]))
 695                 {
 696                         echo fread($ahPipes[1], 4096);
 697                 }
 698                 fclose($ahPipes[1]);
 699
 700                 proc_close($hProcess);
 701         }
 702
 703         function pgsqlRunScript($sScript)
 704         {
 705                 // Convert database DSN to psql parameters
 706                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 707                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 708                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 709                 $aDescriptors = array(
 710                         0 => array('pipe', 'r'),
 711                         1 => STDOUT,
 712                         2 => STDERR
 713                 );
 714                 $ahPipes = null;
 715                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 716                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 717
 718                 while(strlen($sScript))
 719                 {
 720                         $written = fwrite($ahPipes[0], $sScript);
 721                         $sScript = substr($sScript, $written);
 722                 }
 723                 fclose($ahPipes[0]);
 724                 proc_close($hProcess);
 725         }
 726
 727         function pgsqlRunRestoreData($sDumpFile)
 728         {
 729                 // Convert database DSN to psql parameters
 730                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 731                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 732                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 733
 734                 $aDescriptors = array(
 735                         0 => array('pipe', 'r'),
 736                         1 => array('pipe', 'w'),
 737                         2 => array('file', '/dev/null', 'a')
 738                 );
 739                 $ahPipes = null;
 740                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 741                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 742
 743                 fclose($ahPipes[0]);
 744
 745                 // TODO: error checking
 746                 while(!feof($ahPipes[1]))
 747                 {
 748                         echo fread($ahPipes[1], 4096);
 749                 }
 750                 fclose($ahPipes[1]);
 751
 752                 proc_close($hProcess);
 753         }
 754
 755         function pgsqlRunDropAndRestore($sDumpFile)
 756         {
 757                 // Convert database DSN to psql parameters
 758                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 759                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 760                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 761
 762                 $aDescriptors = array(
 763                         0 => array('pipe', 'r'),
 764                         1 => array('pipe', 'w'),
 765                         2 => array('file', '/dev/null', 'a')
 766                 );
 767                 $ahPipes = null;
 768                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 769                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 770
 771                 fclose($ahPipes[0]);
 772
 773                 // TODO: error checking
 774                 while(!feof($ahPipes[1]))
 775                 {
 776                         echo fread($ahPipes[1], 4096);
 777                 }
 778                 fclose($ahPipes[1]);
 779
 780                 proc_close($hProcess);
 781         }
 782
 783         function passthruCheckReturn($cmd)
 784         {
 785                 $result = -1;
 786                 passthru($cmd, $result);
 787                 if ($result != 0) fail('Error executing external command: '.$cmd);
 788         }