utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse opertions during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // This is a pretty hard core defult - the number of processors in the box - 1
  47         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  48         if ($iInstances < 1)
  49         {
  50                 $iInstances = 1;
  51                 echo "WARNING: resetting threads to $iInstances\n";
  52         }
  53         if ($iInstances > getProcessorCount())
  54         {
  55                 $iInstances = getProcessorCount();
  56                 echo "WARNING: resetting threads to $iInstances\n";
  57         }
  58
  59         // Assume we can steal all the cache memory in the box (unless told otherwise)
  60         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  61         if ($iCacheMemory > getTotalMemoryMB())
  62         {
  63                 $iCacheMemory = getCacheMemoryMB();
  64                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  65         }
  66
  67         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  68         {
  69                 $sBaseFile = basename($aCMDResult['osm-file']);
  70                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  71                 {
  72                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  73                         $iTime -= (60*60*24);
  74                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  75                 }
  76         }
  77         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  78         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  79
  80         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  81         {
  82                 echo "Create DB\n";
  83                 $bDidSomething = true;
  84                 $oDB =& DB::connect(CONST_Database_DSN, false);
  85                 if (!PEAR::isError($oDB))
  86                 {
  87                         fail('database already exists ('.CONST_Database_DSN.')');
  88                 }
  89                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  90         }
  91
  92         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  93         {
  94                 echo "Create DB (2)\n";
  95                 $bDidSomething = true;
  96                 // TODO: path detection, detection memory, etc.
  97
  98                 $oDB =& getDB();
  99                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 100                 $pgver = (float) CONST_Postgresql_Version;
 101                 if ($pgver < 9.1) {
 102                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 103                 } else {
 104                         pgsqlRunScript('CREATE EXTENSION hstore');
 105                 }
 106                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 107                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 108                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 109                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 110                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 111                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 112                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 113                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 114                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 115                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 116         }
 117
 118         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 119         {
 120                 echo "Import\n";
 121                 $bDidSomething = true;
 122
 123                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 124                 if (!file_exists($osm2pgsql))
 125                 {
 126                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 127                         fail("osm2pgsql not found in '$osm2pgsql'");
 128                 }
 129                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 130                 $osm2pgsql .= ' -C '.$iCacheMemory;
 131                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 132                 passthru($osm2pgsql);
 133
 134                 $oDB =& getDB();
 135                 $x = $oDB->getRow('select * from place limit 1');
 136                 if (PEAR::isError($x)) {
 137                         fail($x->getMessage());
 138                 }
 139                 if (!$x) fail('No Data');
 140         }
 141
 142         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 143         {
 144                 echo "Functions\n";
 145                 $bDidSomething = true;
 146                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 147                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 148                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 149                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 150                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 151                 pgsqlRunScript($sTemplate);
 152         }
 153
 154         if ($aCMDResult['create-minimal-tables'])
 155         {
 156                 echo "Minimal Tables\n";
 157                 $bDidSomething = true;
 158                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 159
 160                 $sScript = '';
 161
 162                 // Backstop the import process - easliest possible import id
 163                 $sScript .= "insert into import_npi_log values (18022);\n";
 164
 165                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 166                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 167
 168                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 169                 {
 170                         list($sClass, $sType) = explode(' ', trim($sLine));
 171                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 172                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 173
 174                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 175                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 176
 177                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 178                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 179                 }
 180                 fclose($hFile);
 181                 pgsqlRunScript($sScript);
 182         }
 183
 184         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 185         {
 186                 echo "Tables\n";
 187                 $bDidSomething = true;
 188                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 189
 190                 // re-run the functions
 191                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 192                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 193                 pgsqlRunScript($sTemplate);
 194         }
 195
 196         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 197         {
 198                 echo "Partitions\n";
 199                 $bDidSomething = true;
 200                 $oDB =& getDB();
 201                 $sSQL = 'select partition from country_name order by country_code';
 202                 $aPartitions = $oDB->getCol($sSQL);
 203                 if (PEAR::isError($aPartitions))
 204                 {
 205                         fail($aPartitions->getMessage());
 206                 }
 207                 $aPartitions[] = 0;
 208
 209                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 210                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 211                 foreach($aMatches as $aMatch)
 212                 {
 213                         $sResult = '';
 214                         foreach($aPartitions as $sPartitionName)
 215                         {
 216                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 217                         }
 218                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 219                 }
 220
 221                 pgsqlRunScript($sTemplate);
 222         }
 223
 224         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 225         {
 226                 $bDidSomething = true;
 227                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 228                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 229                 if (file_exists($sWikiArticlesFile))
 230                 {
 231                         echo "Importing wikipedia articles...";
 232                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 233                         echo "...done\n";
 234                 }
 235                 else
 236                 {
 237                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 238                 }
 239                 if (file_exists($sWikiRedirectsFile))
 240                 {
 241                         echo "Importing wikipedia redirects...";
 242                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 243                         echo "...done\n";
 244                 }
 245                 else
 246                 {
 247                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 248                 }
 249         }
 250
 251
 252         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 253         {
 254                 echo "Load Data\n";
 255                 $bDidSomething = true;
 256
 257                 $oDB =& getDB();
 258                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 259                 echo '.';
 260                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 261                 echo '.';
 262                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 263                 echo '.';
 264                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 265                 echo '.';
 266                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 267                 echo '.';
 268                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 269                 echo '.';
 270                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 271                 echo '.';
 272                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 273                 echo '.';
 274                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 275                 echo '.';
 276
 277                 // pre-create the word list
 278                 if (!$aCMDResult['disable-token-precalc'])
 279                 {
 280                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 281                         echo '.';
 282                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 283                         echo '.';
 284                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 285                         echo '.';
 286                 }
 287
 288                 $aDBInstances = array();
 289                 for($i = 0; $i < $iInstances; $i++)
 290                 {
 291                         $aDBInstances[$i] =& getDB(true);
 292                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 293                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 294                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 295                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 296                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 297                 }
 298                 $bAnyBusy = true;
 299                 while($bAnyBusy)
 300                 {
 301                         $bAnyBusy = false;
 302                         for($i = 0; $i < $iInstances; $i++)
 303                         {
 304                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 305                         }
 306                         sleep(1);
 307                         echo '.';
 308                 }
 309                 echo "\n";
 310                 echo "Reanalysing database...\n";
 311                 pgsqlRunScript('ANALYSE');
 312         }
 313
 314         if ($aCMDResult['create-roads'])
 315         {
 316                 $bDidSomething = true;
 317
 318                 $oDB =& getDB();
 319                 $aDBInstances = array();
 320                 for($i = 0; $i < $iInstances; $i++)
 321                 {
 322                         $aDBInstances[$i] =& getDB(true);
 323                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 324                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 325                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 326                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 327                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 328                 }
 329                 $bAnyBusy = true;
 330                 while($bAnyBusy)
 331                 {
 332                         $bAnyBusy = false;
 333                         for($i = 0; $i < $iInstances; $i++)
 334                         {
 335                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 336                         }
 337                         sleep(1);
 338                         echo '.';
 339                 }
 340                 echo "\n";
 341         }
 342
 343         if ($aCMDResult['import-tiger-data'])
 344         {
 345                 $bDidSomething = true;
 346
 347                 $aDBInstances = array();
 348                 for($i = 0; $i < $iInstances; $i++)
 349                 {
 350                         $aDBInstances[$i] =& getDB(true);
 351                 }
 352
 353                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 354                 {
 355                         echo $sFile.': ';
 356                         $hFile = fopen($sFile, "r");
 357                         $sSQL = fgets($hFile, 100000);
 358                         $iLines = 0;
 359
 360                         while(true)
 361                         {
 362                                 for($i = 0; $i < $iInstances; $i++)
 363                                 {
 364                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 365                                         {
 366                                                 while(pg_get_result($aDBInstances[$i]->connection));
 367                                                 $sSQL = fgets($hFile, 100000);
 368                                                 if (!$sSQL) break 2;
 369                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 370                                                 $iLines++;
 371                                                 if ($iLines == 1000)
 372                                                 {
 373                                                         echo ".";
 374                                                         $iLines = 0;
 375                                                 }
 376                                         }
 377                                 }
 378                                 usleep(10);
 379                         }
 380
 381                         fclose($hFile);
 382
 383                         $bAnyBusy = true;
 384                         while($bAnyBusy)
 385                         {
 386                                 $bAnyBusy = false;
 387                                 for($i = 0; $i < $iInstances; $i++)
 388                                 {
 389                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 390                                 }
 391                                 usleep(10);
 392                         }
 393                         echo "\n";
 394                 }
 395         }
 396
 397         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 398         {
 399                 $bDidSomething = true;
 400                 $oDB =& getDB();
 401                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 402                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 403                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 404                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 405                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 406                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 407                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 408
 409                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 410                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 411                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 412                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 413         }
 414
 415         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 416         {
 417                 $bDidSomething = true;
 418                 $oDB =& getDB();
 419
 420                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 421                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 422                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 423
 424                 $sDate = $aCMDResult['osmosis-init-date'];
 425                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 426                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 427                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 428                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 429                 $sURL .= '&stream=minute';
 430                 echo "Getting state file: $sURL\n";
 431                 $sStateFile = file_get_contents($sURL);
 432                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 433                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 434                 echo "Updating DB status\n";
 435                 pg_query($oDB->connection, 'TRUNCATE import_status');
 436                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 437                 pg_query($oDB->connection, $sSQL);
 438
 439         }
 440
 441         if ($aCMDResult['index'] || $aCMDResult['all'])
 442         {
 443                 $bDidSomething = true;
 444                 $sOutputFile = '';
 445                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 446                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 447                 passthru($sBaseCmd.' -R 4');
 448                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 449                 passthru($sBaseCmd.' -r 5 -R 25');
 450                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 451                 passthru($sBaseCmd.' -r 26');
 452         }
 453
 454         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 455         {
 456                 echo "Search indices\n";
 457                 $bDidSomething = true;
 458                 $oDB =& getDB();
 459                 $sSQL = 'select partition from country_name order by country_code';
 460                 $aPartitions = $oDB->getCol($sSQL);
 461                 if (PEAR::isError($aPartitions))
 462                 {
 463                         fail($aPartitions->getMessage());
 464                 }
 465                 $aPartitions[] = 0;
 466
 467                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 468                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 469                 foreach($aMatches as $aMatch)
 470                 {
 471                         $sResult = '';
 472                         foreach($aPartitions as $sPartitionName)
 473                         {
 474                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 475                         }
 476                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 477                 }
 478
 479                 pgsqlRunScript($sTemplate);
 480         }
 481
 482         if (isset($aCMDResult['create-website']))
 483         {
 484                 $bDidSomething = true;
 485                 $sTargetDir = $aCMDResult['create-website'];
 486                 if (!is_dir($sTargetDir))
 487                 {
 488                         echo "You must create the website directory before calling this function.\n";
 489                         fail("Target directory does not exist.");
 490                 }
 491
 492                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 493                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 494                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 495                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 496                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 497                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 498                 echo "Symlinks created\n";
 499         }
 500
 501         if (!$bDidSomething)
 502         {
 503                 showUsage($aCMDOptions, true);
 504         }
 505
 506         function pgsqlRunScriptFile($sFilename)
 507         {
 508                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 509
 510                 // Convert database DSN to psql paramaters
 511                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 512                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 513                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 514
 515                 $aDescriptors = array(
 516                         0 => array('pipe', 'r'),
 517                         1 => array('pipe', 'w'),
 518                         2 => array('file', '/dev/null', 'a')
 519                 );
 520                 $ahPipes = null;
 521                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 522                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 523
 524                 fclose($ahPipes[0]);
 525
 526                 // TODO: error checking
 527                 while(!feof($ahPipes[1]))
 528                 {
 529                         echo fread($ahPipes[1], 4096);
 530                 }
 531                 fclose($ahPipes[1]);
 532
 533                 proc_close($hProcess);
 534         }
 535
 536         function pgsqlRunScript($sScript)
 537         {
 538                 // Convert database DSN to psql paramaters
 539                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 540                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 541                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 542                 $aDescriptors = array(
 543                         0 => array('pipe', 'r'),
 544                         1 => STDOUT,
 545                         2 => STDERR
 546                 );
 547                 $ahPipes = null;
 548                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 549                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 550
 551                 while(strlen($sScript))
 552                 {
 553                         $written = fwrite($ahPipes[0], $sScript);
 554                         $sScript = substr($sScript, $written);
 555                 }
 556                 fclose($ahPipes[0]);
 557                 proc_close($hProcess);
 558         }
 559
 560         function pgsqlRunRestoreData($sDumpFile)
 561         {
 562                 // Convert database DSN to psql paramaters
 563                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 564                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 565                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 566
 567                 $aDescriptors = array(
 568                         0 => array('pipe', 'r'),
 569                         1 => array('pipe', 'w'),
 570                         2 => array('file', '/dev/null', 'a')
 571                 );
 572                 $ahPipes = null;
 573                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 574                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 575
 576                 fclose($ahPipes[0]);
 577
 578                 // TODO: error checking
 579                 while(!feof($ahPipes[1]))
 580                 {
 581                         echo fread($ahPipes[1], 4096);
 582                 }
 583                 fclose($ahPipes[1]);
 584
 585                 proc_close($hProcess);
 586         }
 587
 588         function pgsqlRunDropAndRestore($sDumpFile)
 589         {
 590                 // Convert database DSN to psql paramaters
 591                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 592                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 593                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 594
 595                 $aDescriptors = array(
 596                         0 => array('pipe', 'r'),
 597                         1 => array('pipe', 'w'),
 598                         2 => array('file', '/dev/null', 'a')
 599                 );
 600                 $ahPipes = null;
 601                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 602                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 603
 604                 fclose($ahPipes[0]);
 605
 606                 // TODO: error checking
 607                 while(!feof($ahPipes[1]))
 608                 {
 609                         echo fread($ahPipes[1], 4096);
 610                 }
 611                 fclose($ahPipes[1]);
 612
 613                 proc_close($hProcess);
 614         }