utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
  28                 array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
  29                 array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
  30                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  31                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  32                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  33                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  34                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  35                 array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
  36                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  38                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  39                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  40                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  41                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  42         );
  43         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  44
  45         $bDidSomething = false;
  46
  47         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  48         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  49         {
  50                 if (!isset($aCMDResult['osm-file']))
  51                 {
  52                         fail('missing --osm-file for data import');
  53                 }
  54
  55                 if (!file_exists($aCMDResult['osm-file']))
  56                 {
  57                         fail('the path supplied to --osm-file does not exist');
  58                 }
  59
  60                 if (!is_readable($aCMDResult['osm-file']))
  61                 {
  62                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  63                 }
  64         }
  65
  66
  67         // This is a pretty hard core default - the number of processors in the box - 1
  68         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  69         if ($iInstances < 1)
  70         {
  71                 $iInstances = 1;
  72                 echo "WARNING: resetting threads to $iInstances\n";
  73         }
  74         if ($iInstances > getProcessorCount())
  75         {
  76                 $iInstances = getProcessorCount();
  77                 echo "WARNING: resetting threads to $iInstances\n";
  78         }
  79
  80         // Assume we can steal all the cache memory in the box (unless told otherwise)
  81         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  82         if ($iCacheMemory > getTotalMemoryMB())
  83         {
  84                 $iCacheMemory = getCacheMemoryMB();
  85                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  86         }
  87
  88         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  89         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  90
  91         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  92         {
  93                 echo "Create DB\n";
  94                 $bDidSomething = true;
  95                 $oDB =& DB::connect(CONST_Database_DSN, false);
  96                 if (!PEAR::isError($oDB))
  97                 {
  98                         fail('database already exists ('.CONST_Database_DSN.')');
  99                 }
 100                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 101         }
 102
 103         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 104         {
 105                 echo "Setup DB\n";
 106                 $bDidSomething = true;
 107                 // TODO: path detection, detection memory, etc.
 108
 109                 $oDB =& getDB();
 110
 111                 $sVersionString = $oDB->getOne('select version()');
 112                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 113                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 114                 {
 115                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 116                         exit;
 117                 }
 118
 119                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 124                 } else {
 125                         pgsqlRunScript('CREATE EXTENSION hstore');
 126                 }
 127
 128                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 129                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 130                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 131                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 132                 {
 133                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 134                         exit;
 135                 }
 136
 137                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 138                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 139                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 140                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 141                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 142                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 143                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 144                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 145
 146                 if ($aCMDResult['no-partitions'])
 147                 {
 148                         pgsqlRunScript('update country_name set partition = 0');
 149                 }
 150         }
 151
 152         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 153         {
 154                 echo "Import\n";
 155                 $bDidSomething = true;
 156
 157                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 158                 if (!file_exists($osm2pgsql))
 159                 {
 160                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 161                         fail("osm2pgsql not found in '$osm2pgsql'");
 162                 }
 163
 164                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 165                 {
 166                         $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 167                 }
 168                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 169                 $osm2pgsql .= ' -C '.$iCacheMemory;
 170                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 171                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 172                 passthruCheckReturn($osm2pgsql);
 173
 174                 $oDB =& getDB();
 175                 $x = $oDB->getRow('select * from place limit 1');
 176                 if (PEAR::isError($x)) {
 177                         fail($x->getMessage());
 178                 }
 179                 if (!$x) fail('No Data');
 180         }
 181
 182         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 183         {
 184                 echo "Functions\n";
 185                 $bDidSomething = true;
 186                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 187                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 188                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 189                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 190                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 191                 pgsqlRunScript($sTemplate);
 192         }
 193
 194         if ($aCMDResult['create-minimal-tables'])
 195         {
 196                 echo "Minimal Tables\n";
 197                 $bDidSomething = true;
 198                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 199
 200                 $sScript = '';
 201
 202                 // Backstop the import process - easliest possible import id
 203                 $sScript .= "insert into import_npi_log values (18022);\n";
 204
 205                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 206                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 207
 208                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 209                 {
 210                         list($sClass, $sType) = explode(' ', trim($sLine));
 211                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 212                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 213
 214                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 215                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 216
 217                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 218                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 219                 }
 220                 fclose($hFile);
 221                 pgsqlRunScript($sScript);
 222         }
 223
 224         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 225         {
 226                 echo "Tables\n";
 227                 $bDidSomething = true;
 228                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 229
 230                 // re-run the functions
 231                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 232                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 233                 pgsqlRunScript($sTemplate);
 234         }
 235
 236         if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 237         {
 238                 echo "Partition Tables\n";
 239                 $bDidSomething = true;
 240                 $oDB =& getDB();
 241                 $sSQL = 'select distinct partition from country_name';
 242                 $aPartitions = $oDB->getCol($sSQL);
 243                 if (PEAR::isError($aPartitions))
 244                 {
 245                         fail($aPartitions->getMessage());
 246                 }
 247                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 248
 249                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
 250                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 251                 foreach($aMatches as $aMatch)
 252                 {
 253                         $sResult = '';
 254                         foreach($aPartitions as $sPartitionName)
 255                         {
 256                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 257                         }
 258                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 259                 }
 260
 261                 pgsqlRunScript($sTemplate);
 262         }
 263
 264
 265         if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
 266         {
 267                 echo "Partition Functions\n";
 268                 $bDidSomething = true;
 269                 $oDB =& getDB();
 270                 $sSQL = 'select distinct partition from country_name';
 271                 $aPartitions = $oDB->getCol($sSQL);
 272                 if (PEAR::isError($aPartitions))
 273                 {
 274                         fail($aPartitions->getMessage());
 275                 }
 276                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 277
 278                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 279                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 280                 foreach($aMatches as $aMatch)
 281                 {
 282                         $sResult = '';
 283                         foreach($aPartitions as $sPartitionName)
 284                         {
 285                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 286                         }
 287                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 288                 }
 289
 290                 pgsqlRunScript($sTemplate);
 291         }
 292
 293         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 294         {
 295                 $bDidSomething = true;
 296                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 297                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 298                 if (file_exists($sWikiArticlesFile))
 299                 {
 300                         echo "Importing wikipedia articles...";
 301                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 302                         echo "...done\n";
 303                 }
 304                 else
 305                 {
 306                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 307                 }
 308                 if (file_exists($sWikiRedirectsFile))
 309                 {
 310                         echo "Importing wikipedia redirects...";
 311                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 312                         echo "...done\n";
 313                 }
 314                 else
 315                 {
 316                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 317                 }
 318         }
 319
 320
 321         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 322         {
 323                 echo "Drop old Data\n";
 324                 $bDidSomething = true;
 325
 326                 $oDB =& getDB();
 327                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 328                 echo '.';
 329                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 330                 echo '.';
 331                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 332                 echo '.';
 333                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 334                 echo '.';
 335                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 336                 echo '.';
 337                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 338                 echo '.';
 339                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 340                 echo '.';
 341                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 342                 echo '.';
 343                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 344                 echo '.';
 345
 346                 $sSQL = 'select distinct partition from country_name';
 347                 $aPartitions = $oDB->getCol($sSQL);
 348                 if (PEAR::isError($aPartitions))
 349                 {
 350                         fail($aPartitions->getMessage());
 351                 }
 352                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 353                 foreach($aPartitions as $sPartition)
 354                 {
 355                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 356                         echo '.';
 357                 }
 358
 359                 // used by getorcreate_word_id to ignore frequent partial words
 360                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 361                 echo ".\n";
 362
 363                 // pre-create the word list
 364                 if (!$aCMDResult['disable-token-precalc'])
 365                 {
 366                         echo "Loading word list\n";
 367                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 368                 }
 369
 370                 echo "Load Data\n";
 371                 $aDBInstances = array();
 372                 for($i = 0; $i < $iInstances; $i++)
 373                 {
 374                         $aDBInstances[$i] =& getDB(true);
 375                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 376                         $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 377                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 378                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 379                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 380                 }
 381                 $bAnyBusy = true;
 382                 while($bAnyBusy)
 383                 {
 384                         $bAnyBusy = false;
 385                         for($i = 0; $i < $iInstances; $i++)
 386                         {
 387                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 388                         }
 389                         sleep(1);
 390                         echo '.';
 391                 }
 392                 echo "\n";
 393                 echo "Reanalysing database...\n";
 394                 pgsqlRunScript('ANALYSE');
 395         }
 396
 397         if ($aCMDResult['create-roads'])
 398         {
 399                 $bDidSomething = true;
 400
 401                 $oDB =& getDB();
 402                 $aDBInstances = array();
 403                 for($i = 0; $i < $iInstances; $i++)
 404                 {
 405                         $aDBInstances[$i] =& getDB(true);
 406                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 407                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 408                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 409                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 410                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 411                 }
 412                 $bAnyBusy = true;
 413                 while($bAnyBusy)
 414                 {
 415                         $bAnyBusy = false;
 416                         for($i = 0; $i < $iInstances; $i++)
 417                         {
 418                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 419                         }
 420                         sleep(1);
 421                         echo '.';
 422                 }
 423                 echo "\n";
 424         }
 425
 426         if ($aCMDResult['import-tiger-data'])
 427         {
 428                 $bDidSomething = true;
 429
 430                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 431
 432                 $aDBInstances = array();
 433                 for($i = 0; $i < $iInstances; $i++)
 434                 {
 435                         $aDBInstances[$i] =& getDB(true);
 436                 }
 437
 438                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 439                 {
 440                         echo $sFile.': ';
 441                         $hFile = fopen($sFile, "r");
 442                         $sSQL = fgets($hFile, 100000);
 443                         $iLines = 0;
 444
 445                         while(true)
 446                         {
 447                                 for($i = 0; $i < $iInstances; $i++)
 448                                 {
 449                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 450                                         {
 451                                                 while(pg_get_result($aDBInstances[$i]->connection));
 452                                                 $sSQL = fgets($hFile, 100000);
 453                                                 if (!$sSQL) break 2;
 454                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 455                                                 $iLines++;
 456                                                 if ($iLines == 1000)
 457                                                 {
 458                                                         echo ".";
 459                                                         $iLines = 0;
 460                                                 }
 461                                         }
 462                                 }
 463                                 usleep(10);
 464                         }
 465
 466                         fclose($hFile);
 467
 468                         $bAnyBusy = true;
 469                         while($bAnyBusy)
 470                         {
 471                                 $bAnyBusy = false;
 472                                 for($i = 0; $i < $iInstances; $i++)
 473                                 {
 474                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 475                                 }
 476                                 usleep(10);
 477                         }
 478                         echo "\n";
 479                 }
 480
 481                 echo "Creating indexes\n";
 482                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 483         }
 484
 485         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 486         {
 487                 $bDidSomething = true;
 488                 $oDB =& getDB();
 489                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 490                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 491                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 492                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 493                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 494                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 495                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 496
 497                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 498                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 499                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 500                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 501         }
 502
 503         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 504         {
 505                 $bDidSomething = true;
 506                 $oDB =& getDB();
 507
 508                 if (!file_exists(CONST_Osmosis_Binary))
 509                 {
 510                         echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 511                         if (!$aCMDResult['all'])
 512                         {
 513                                 fail("osmosis not found in '".CONST_Osmosis_Binary."'");
 514                         }
 515                 }
 516                 else
 517                 {
 518                         if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 519                         {
 520                                 echo "settings/configuration.txt already exists\n";
 521                         }
 522                         else
 523                         {
 524                                 passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 525                                 // update osmosis configuration.txt with our settings
 526                                 passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
 527                                 passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
 528                         }
 529
 530                         // Find the last node in the DB
 531                         $iLastOSMID = $oDB->getOne("select max(id) from planet_osm_nodes");
 532
 533                         // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 534                         $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
 535                         $sLastNodeXML = file_get_contents($sLastNodeURL);
 536                         preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 537                         $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 538
 539                         // Search for the correct state file - uses file timestamps so need to sort by date descending
 540                         $sRepURL = CONST_Replication_Url."/";
 541                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 542                         // download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
 543                         // planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
 544                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 545                         $aPrevRepMatch = false;
 546                         foreach($aRepMatches as $aRepMatch)
 547                         {
 548                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 549                                 $aPrevRepMatch = $aRepMatch;
 550                         }
 551                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 552
 553                         $sRepURL .= $aRepMatch[1];
 554                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 555                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 556                         $aPrevRepMatch = false;
 557                         foreach($aRepMatches as $aRepMatch)
 558                         {
 559                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 560                                 $aPrevRepMatch = $aRepMatch;
 561                         }
 562                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 563
 564                         $sRepURL .= $aRepMatch[1];
 565                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 566                         preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 567                         $aPrevRepMatch = false;
 568                         foreach($aRepMatches as $aRepMatch)
 569                         {
 570                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 571                                 $aPrevRepMatch = $aRepMatch;
 572                         }
 573                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 574
 575                         $sRepURL .= $aRepMatch[1].'.state.txt';
 576                         echo "Getting state file: $sRepURL\n";
 577                         $sStateFile = file_get_contents($sRepURL);
 578                         if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 579                         file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 580                         echo "Updating DB status\n";
 581                         pg_query($oDB->connection, 'TRUNCATE import_status');
 582                         $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 583                         pg_query($oDB->connection, $sSQL);
 584                 }
 585         }
 586
 587         if ($aCMDResult['index'] || $aCMDResult['all'])
 588         {
 589                 $bDidSomething = true;
 590                 $sOutputFile = '';
 591                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 592                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 593                 passthruCheckReturn($sBaseCmd.' -R 4');
 594                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 595                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 596                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 597                 passthruCheckReturn($sBaseCmd.' -r 26');
 598         }
 599
 600         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 601         {
 602                 echo "Search indices\n";
 603                 $bDidSomething = true;
 604                 $oDB =& getDB();
 605                 $sSQL = 'select distinct partition from country_name';
 606                 $aPartitions = $oDB->getCol($sSQL);
 607                 if (PEAR::isError($aPartitions))
 608                 {
 609                         fail($aPartitions->getMessage());
 610                 }
 611                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 612
 613                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 614                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 615                 foreach($aMatches as $aMatch)
 616                 {
 617                         $sResult = '';
 618                         foreach($aPartitions as $sPartitionName)
 619                         {
 620                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 621                         }
 622                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 623                 }
 624
 625                 pgsqlRunScript($sTemplate);
 626         }
 627
 628         if (isset($aCMDResult['create-website']))
 629         {
 630                 $bDidSomething = true;
 631                 $sTargetDir = $aCMDResult['create-website'];
 632                 if (!is_dir($sTargetDir))
 633                 {
 634                         echo "You must create the website directory before calling this function.\n";
 635                         fail("Target directory does not exist.");
 636                 }
 637
 638                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 639                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 640                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 641                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 642                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 643                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 644                 @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php');
 645                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 646                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 647                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 648                 echo "Symlinks created\n";
 649
 650                 $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
 651                 if (!$sTestFile)
 652                 {
 653                         echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
 654                         echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
 655                 }
 656         }
 657
 658         if (!$bDidSomething)
 659         {
 660                 showUsage($aCMDOptions, true);
 661         }
 662
 663         function pgsqlRunScriptFile($sFilename)
 664         {
 665                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 666
 667                 // Convert database DSN to psql parameters
 668                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 669                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 670                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 671
 672                 $aDescriptors = array(
 673                         0 => array('pipe', 'r'),
 674                         1 => array('pipe', 'w'),
 675                         2 => array('file', '/dev/null', 'a')
 676                 );
 677                 $ahPipes = null;
 678                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 679                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 680
 681                 fclose($ahPipes[0]);
 682
 683                 // TODO: error checking
 684                 while(!feof($ahPipes[1]))
 685                 {
 686                         echo fread($ahPipes[1], 4096);
 687                 }
 688                 fclose($ahPipes[1]);
 689
 690                 proc_close($hProcess);
 691         }
 692
 693         function pgsqlRunScript($sScript)
 694         {
 695                 // Convert database DSN to psql parameters
 696                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 697                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 698                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 699                 $aDescriptors = array(
 700                         0 => array('pipe', 'r'),
 701                         1 => STDOUT,
 702                         2 => STDERR
 703                 );
 704                 $ahPipes = null;
 705                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 706                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 707
 708                 while(strlen($sScript))
 709                 {
 710                         $written = fwrite($ahPipes[0], $sScript);
 711                         $sScript = substr($sScript, $written);
 712                 }
 713                 fclose($ahPipes[0]);
 714                 proc_close($hProcess);
 715         }
 716
 717         function pgsqlRunRestoreData($sDumpFile)
 718         {
 719                 // Convert database DSN to psql parameters
 720                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 721                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 722                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 723
 724                 $aDescriptors = array(
 725                         0 => array('pipe', 'r'),
 726                         1 => array('pipe', 'w'),
 727                         2 => array('file', '/dev/null', 'a')
 728                 );
 729                 $ahPipes = null;
 730                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 731                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 732
 733                 fclose($ahPipes[0]);
 734
 735                 // TODO: error checking
 736                 while(!feof($ahPipes[1]))
 737                 {
 738                         echo fread($ahPipes[1], 4096);
 739                 }
 740                 fclose($ahPipes[1]);
 741
 742                 proc_close($hProcess);
 743         }
 744
 745         function pgsqlRunDropAndRestore($sDumpFile)
 746         {
 747                 // Convert database DSN to psql parameters
 748                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 749                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 750                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 751
 752                 $aDescriptors = array(
 753                         0 => array('pipe', 'r'),
 754                         1 => array('pipe', 'w'),
 755                         2 => array('file', '/dev/null', 'a')
 756                 );
 757                 $ahPipes = null;
 758                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 759                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 760
 761                 fclose($ahPipes[0]);
 762
 763                 // TODO: error checking
 764                 while(!feof($ahPipes[1]))
 765                 {
 766                         echo fread($ahPipes[1], 4096);
 767                 }
 768                 fclose($ahPipes[1]);
 769
 770                 proc_close($hProcess);
 771         }
 772
 773         function passthruCheckReturn($cmd)
 774         {
 775                 $result = -1;
 776                 passthru($cmd, $result);
 777                 if ($result != 0) fail('Error executing external command: '.$cmd);
 778         }