utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // This is a pretty hard core default - the number of processors in the box - 1
  47         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  48         if ($iInstances < 1)
  49         {
  50                 $iInstances = 1;
  51                 echo "WARNING: resetting threads to $iInstances\n";
  52         }
  53         if ($iInstances > getProcessorCount())
  54         {
  55                 $iInstances = getProcessorCount();
  56                 echo "WARNING: resetting threads to $iInstances\n";
  57         }
  58
  59         // Assume we can steal all the cache memory in the box (unless told otherwise)
  60         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  61         if ($iCacheMemory > getTotalMemoryMB())
  62         {
  63                 $iCacheMemory = getCacheMemoryMB();
  64                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  65         }
  66
  67         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  68         {
  69                 $sBaseFile = basename($aCMDResult['osm-file']);
  70                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  71                 {
  72                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  73                         $iTime -= (60*60*24);
  74                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  75                 }
  76         }
  77         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  78         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  79
  80         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  81         {
  82                 echo "Create DB\n";
  83                 $bDidSomething = true;
  84                 $oDB =& DB::connect(CONST_Database_DSN, false);
  85                 if (!PEAR::isError($oDB))
  86                 {
  87                         fail('database already exists ('.CONST_Database_DSN.')');
  88                 }
  89                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  90         }
  91
  92         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  93         {
  94                 echo "Create DB (2)\n";
  95                 $bDidSomething = true;
  96                 // TODO: path detection, detection memory, etc.
  97
  98                 $oDB =& getDB();
  99                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 100                 $pgver = (float) CONST_Postgresql_Version;
 101                 if ($pgver < 9.1) {
 102                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 103                 } else {
 104                         pgsqlRunScript('CREATE EXTENSION hstore');
 105                 }
 106                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 107                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 108                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 109                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 110                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 111                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 112                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 113                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 114                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 115                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 116         }
 117
 118         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 119         {
 120                 echo "Import\n";
 121                 $bDidSomething = true;
 122
 123                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 124                 if (!file_exists($osm2pgsql))
 125                 {
 126                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 127                         fail("osm2pgsql not found in '$osm2pgsql'");
 128                 }
 129                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 130                 $osm2pgsql .= ' -C '.$iCacheMemory;
 131                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 132                 passthruCheckReturn($osm2pgsql);
 133
 134                 $oDB =& getDB();
 135                 $x = $oDB->getRow('select * from place limit 1');
 136                 if (PEAR::isError($x)) {
 137                         fail($x->getMessage());
 138                 }
 139                 if (!$x) fail('No Data');
 140         }
 141
 142         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 143         {
 144                 echo "Functions\n";
 145                 $bDidSomething = true;
 146                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 147                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 148                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 149                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 150                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 151                 pgsqlRunScript($sTemplate);
 152         }
 153
 154         if ($aCMDResult['create-minimal-tables'])
 155         {
 156                 echo "Minimal Tables\n";
 157                 $bDidSomething = true;
 158                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 159
 160                 $sScript = '';
 161
 162                 // Backstop the import process - easliest possible import id
 163                 $sScript .= "insert into import_npi_log values (18022);\n";
 164
 165                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 166                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 167
 168                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 169                 {
 170                         list($sClass, $sType) = explode(' ', trim($sLine));
 171                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 172                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 173
 174                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 175                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 176
 177                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 178                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 179                 }
 180                 fclose($hFile);
 181                 pgsqlRunScript($sScript);
 182         }
 183
 184         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 185         {
 186                 echo "Tables\n";
 187                 $bDidSomething = true;
 188                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 189
 190                 // re-run the functions
 191                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 192                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 193                 pgsqlRunScript($sTemplate);
 194         }
 195
 196         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 197         {
 198                 echo "Partitions\n";
 199                 $bDidSomething = true;
 200                 $oDB =& getDB();
 201                 $sSQL = 'select partition from country_name order by country_code';
 202                 $aPartitions = $oDB->getCol($sSQL);
 203                 if (PEAR::isError($aPartitions))
 204                 {
 205                         fail($aPartitions->getMessage());
 206                 }
 207                 $aPartitions[] = 0;
 208
 209                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 210                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 211                 foreach($aMatches as $aMatch)
 212                 {
 213                         $sResult = '';
 214                         foreach($aPartitions as $sPartitionName)
 215                         {
 216                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 217                         }
 218                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 219                 }
 220
 221                 pgsqlRunScript($sTemplate);
 222         }
 223
 224         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 225         {
 226                 $bDidSomething = true;
 227                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 228                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 229                 if (file_exists($sWikiArticlesFile))
 230                 {
 231                         echo "Importing wikipedia articles...";
 232                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 233                         echo "...done\n";
 234                 }
 235                 else
 236                 {
 237                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 238                 }
 239                 if (file_exists($sWikiRedirectsFile))
 240                 {
 241                         echo "Importing wikipedia redirects...";
 242                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 243                         echo "...done\n";
 244                 }
 245                 else
 246                 {
 247                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 248                 }
 249         }
 250
 251
 252         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 253         {
 254                 echo "Load Data\n";
 255                 $bDidSomething = true;
 256
 257                 $oDB =& getDB();
 258                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 259                 echo '.';
 260                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 261                 echo '.';
 262                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 263                 echo '.';
 264                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 265                 echo '.';
 266                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 267                 echo '.';
 268                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 269                 echo '.';
 270                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 271                 echo '.';
 272                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 273                 echo '.';
 274                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 275                 echo '.';
 276
 277                 $sSQL = 'select partition from country_name order by country_code';
 278                 $aPartitions = $oDB->getCol($sSQL);
 279                 if (PEAR::isError($aPartitions))
 280                 {
 281                         fail($aPartitions->getMessage());
 282                 }
 283                 $aPartitions[] = 0;
 284                 foreach($aPartitions as $sPartition)
 285                 {
 286                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 287                         echo '.';
 288                 }
 289
 290                 // pre-create the word list
 291                 if (!$aCMDResult['disable-token-precalc'])
 292                 {
 293                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 294                         echo '.';
 295                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 296                         echo '.';
 297                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 298                         echo '.';
 299                 }
 300
 301                 $aDBInstances = array();
 302                 for($i = 0; $i < $iInstances; $i++)
 303                 {
 304                         $aDBInstances[$i] =& getDB(true);
 305                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 306                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 307                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 308                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 309                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 310                 }
 311                 $bAnyBusy = true;
 312                 while($bAnyBusy)
 313                 {
 314                         $bAnyBusy = false;
 315                         for($i = 0; $i < $iInstances; $i++)
 316                         {
 317                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 318                         }
 319                         sleep(1);
 320                         echo '.';
 321                 }
 322                 echo "\n";
 323                 echo "Reanalysing database...\n";
 324                 pgsqlRunScript('ANALYSE');
 325         }
 326
 327         if ($aCMDResult['create-roads'])
 328         {
 329                 $bDidSomething = true;
 330
 331                 $oDB =& getDB();
 332                 $aDBInstances = array();
 333                 for($i = 0; $i < $iInstances; $i++)
 334                 {
 335                         $aDBInstances[$i] =& getDB(true);
 336                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 337                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 338                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 339                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 340                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 341                 }
 342                 $bAnyBusy = true;
 343                 while($bAnyBusy)
 344                 {
 345                         $bAnyBusy = false;
 346                         for($i = 0; $i < $iInstances; $i++)
 347                         {
 348                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 349                         }
 350                         sleep(1);
 351                         echo '.';
 352                 }
 353                 echo "\n";
 354         }
 355
 356         if ($aCMDResult['import-tiger-data'])
 357         {
 358                 $bDidSomething = true;
 359
 360                 $aDBInstances = array();
 361                 for($i = 0; $i < $iInstances; $i++)
 362                 {
 363                         $aDBInstances[$i] =& getDB(true);
 364                 }
 365
 366                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 367                 {
 368                         echo $sFile.': ';
 369                         $hFile = fopen($sFile, "r");
 370                         $sSQL = fgets($hFile, 100000);
 371                         $iLines = 0;
 372
 373                         while(true)
 374                         {
 375                                 for($i = 0; $i < $iInstances; $i++)
 376                                 {
 377                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 378                                         {
 379                                                 while(pg_get_result($aDBInstances[$i]->connection));
 380                                                 $sSQL = fgets($hFile, 100000);
 381                                                 if (!$sSQL) break 2;
 382                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 383                                                 $iLines++;
 384                                                 if ($iLines == 1000)
 385                                                 {
 386                                                         echo ".";
 387                                                         $iLines = 0;
 388                                                 }
 389                                         }
 390                                 }
 391                                 usleep(10);
 392                         }
 393
 394                         fclose($hFile);
 395
 396                         $bAnyBusy = true;
 397                         while($bAnyBusy)
 398                         {
 399                                 $bAnyBusy = false;
 400                                 for($i = 0; $i < $iInstances; $i++)
 401                                 {
 402                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 403                                 }
 404                                 usleep(10);
 405                         }
 406                         echo "\n";
 407                 }
 408         }
 409
 410         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 411         {
 412                 $bDidSomething = true;
 413                 $oDB =& getDB();
 414                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 415                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 416                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 417                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 418                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 419                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 420                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 421
 422                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 423                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 424                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 425                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 426         }
 427
 428         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 429         {
 430                 $bDidSomething = true;
 431                 $oDB =& getDB();
 432
 433                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 434                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 435                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 436
 437                 $sDate = $aCMDResult['osmosis-init-date'];
 438                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 439                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 440                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 441                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 442                 $sURL .= '&stream=minute';
 443                 echo "Getting state file: $sURL\n";
 444                 $sStateFile = file_get_contents($sURL);
 445                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 446                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 447                 echo "Updating DB status\n";
 448                 pg_query($oDB->connection, 'TRUNCATE import_status');
 449                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 450                 pg_query($oDB->connection, $sSQL);
 451
 452         }
 453
 454         if ($aCMDResult['index'] || $aCMDResult['all'])
 455         {
 456                 $bDidSomething = true;
 457                 $sOutputFile = '';
 458                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 459                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 460                 passthruCheckReturn($sBaseCmd.' -R 4');
 461                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 462                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 463                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 464                 passthruCheckReturn($sBaseCmd.' -r 26');
 465         }
 466
 467         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 468         {
 469                 echo "Search indices\n";
 470                 $bDidSomething = true;
 471                 $oDB =& getDB();
 472                 $sSQL = 'select partition from country_name order by country_code';
 473                 $aPartitions = $oDB->getCol($sSQL);
 474                 if (PEAR::isError($aPartitions))
 475                 {
 476                         fail($aPartitions->getMessage());
 477                 }
 478                 $aPartitions[] = 0;
 479
 480                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 481                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 482                 foreach($aMatches as $aMatch)
 483                 {
 484                         $sResult = '';
 485                         foreach($aPartitions as $sPartitionName)
 486                         {
 487                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 488                         }
 489                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 490                 }
 491
 492                 pgsqlRunScript($sTemplate);
 493         }
 494
 495         if (isset($aCMDResult['create-website']))
 496         {
 497                 $bDidSomething = true;
 498                 $sTargetDir = $aCMDResult['create-website'];
 499                 if (!is_dir($sTargetDir))
 500                 {
 501                         echo "You must create the website directory before calling this function.\n";
 502                         fail("Target directory does not exist.");
 503                 }
 504
 505                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 506                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 507                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 508                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 509                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 510                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 511                 echo "Symlinks created\n";
 512         }
 513
 514         if (!$bDidSomething)
 515         {
 516                 showUsage($aCMDOptions, true);
 517         }
 518
 519         function pgsqlRunScriptFile($sFilename)
 520         {
 521                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 522
 523                 // Convert database DSN to psql parameters
 524                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 525                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 526                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 527
 528                 $aDescriptors = array(
 529                         0 => array('pipe', 'r'),
 530                         1 => array('pipe', 'w'),
 531                         2 => array('file', '/dev/null', 'a')
 532                 );
 533                 $ahPipes = null;
 534                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 535                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 536
 537                 fclose($ahPipes[0]);
 538
 539                 // TODO: error checking
 540                 while(!feof($ahPipes[1]))
 541                 {
 542                         echo fread($ahPipes[1], 4096);
 543                 }
 544                 fclose($ahPipes[1]);
 545
 546                 proc_close($hProcess);
 547         }
 548
 549         function pgsqlRunScript($sScript)
 550         {
 551                 // Convert database DSN to psql parameters
 552                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 553                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 554                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 555                 $aDescriptors = array(
 556                         0 => array('pipe', 'r'),
 557                         1 => STDOUT,
 558                         2 => STDERR
 559                 );
 560                 $ahPipes = null;
 561                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 562                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 563
 564                 while(strlen($sScript))
 565                 {
 566                         $written = fwrite($ahPipes[0], $sScript);
 567                         $sScript = substr($sScript, $written);
 568                 }
 569                 fclose($ahPipes[0]);
 570                 proc_close($hProcess);
 571         }
 572
 573         function pgsqlRunRestoreData($sDumpFile)
 574         {
 575                 // Convert database DSN to psql parameters
 576                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 577                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 578                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 579
 580                 $aDescriptors = array(
 581                         0 => array('pipe', 'r'),
 582                         1 => array('pipe', 'w'),
 583                         2 => array('file', '/dev/null', 'a')
 584                 );
 585                 $ahPipes = null;
 586                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 587                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 588
 589                 fclose($ahPipes[0]);
 590
 591                 // TODO: error checking
 592                 while(!feof($ahPipes[1]))
 593                 {
 594                         echo fread($ahPipes[1], 4096);
 595                 }
 596                 fclose($ahPipes[1]);
 597
 598                 proc_close($hProcess);
 599         }
 600
 601         function pgsqlRunDropAndRestore($sDumpFile)
 602         {
 603                 // Convert database DSN to psql parameters
 604                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 605                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 606                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 607
 608                 $aDescriptors = array(
 609                         0 => array('pipe', 'r'),
 610                         1 => array('pipe', 'w'),
 611                         2 => array('file', '/dev/null', 'a')
 612                 );
 613                 $ahPipes = null;
 614                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 615                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 616
 617                 fclose($ahPipes[0]);
 618
 619                 // TODO: error checking
 620                 while(!feof($ahPipes[1]))
 621                 {
 622                         echo fread($ahPipes[1], 4096);
 623                 }
 624                 fclose($ahPipes[1]);
 625
 626                 proc_close($hProcess);
 627         }
 628
 629         function passthruCheckReturn($cmd)
 630         {
 631                 $result = -1;
 632                 passthru($cmd, $result);
 633                 if ($result != 0) fail('Error executing external command: '.$cmd);
 634         }