utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // This is a pretty hard core default - the number of processors in the box - 1
  47         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  48         if ($iInstances < 1)
  49         {
  50                 $iInstances = 1;
  51                 echo "WARNING: resetting threads to $iInstances\n";
  52         }
  53         if ($iInstances > getProcessorCount())
  54         {
  55                 $iInstances = getProcessorCount();
  56                 echo "WARNING: resetting threads to $iInstances\n";
  57         }
  58
  59         // Assume we can steal all the cache memory in the box (unless told otherwise)
  60         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  61         if ($iCacheMemory > getTotalMemoryMB())
  62         {
  63                 $iCacheMemory = getCacheMemoryMB();
  64                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  65         }
  66
  67         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  68         {
  69                 $sBaseFile = basename($aCMDResult['osm-file']);
  70                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  71                 {
  72                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  73                         $iTime -= (60*60*24);
  74                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  75                 }
  76         }
  77         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  78         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  79
  80         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  81         {
  82                 echo "Create DB\n";
  83                 $bDidSomething = true;
  84                 $oDB =& DB::connect(CONST_Database_DSN, false);
  85                 if (!PEAR::isError($oDB))
  86                 {
  87                         fail('database already exists ('.CONST_Database_DSN.')');
  88                 }
  89                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  90         }
  91
  92         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  93         {
  94                 echo "Create DB (2)\n";
  95                 $bDidSomething = true;
  96                 // TODO: path detection, detection memory, etc.
  97
  98                 $oDB =& getDB();
  99                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 100                 $pgver = (float) CONST_Postgresql_Version;
 101                 if ($pgver < 9.1) {
 102                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 103                 } else {
 104                         pgsqlRunScript('CREATE EXTENSION hstore');
 105                 }
 106                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 107                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 108                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 109                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 110                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 111                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 112                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 113                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 114                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 115                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 116         }
 117
 118         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 119         {
 120                 echo "Import\n";
 121                 $bDidSomething = true;
 122
 123                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 124                 if (!file_exists($osm2pgsql))
 125                 {
 126                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 127                         fail("osm2pgsql not found in '$osm2pgsql'");
 128                 }
 129                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 130                 $osm2pgsql .= ' -C '.$iCacheMemory;
 131                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 132                 passthruCheckReturn($osm2pgsql);
 133
 134                 $oDB =& getDB();
 135                 $x = $oDB->getRow('select * from place limit 1');
 136                 if (PEAR::isError($x)) {
 137                         fail($x->getMessage());
 138                 }
 139                 if (!$x) fail('No Data');
 140         }
 141
 142         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 143         {
 144                 echo "Functions\n";
 145                 $bDidSomething = true;
 146                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 147                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 148                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 149                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 150                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 151                 pgsqlRunScript($sTemplate);
 152         }
 153
 154         if ($aCMDResult['create-minimal-tables'])
 155         {
 156                 echo "Minimal Tables\n";
 157                 $bDidSomething = true;
 158                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 159
 160                 $sScript = '';
 161
 162                 // Backstop the import process - easliest possible import id
 163                 $sScript .= "insert into import_npi_log values (18022);\n";
 164
 165                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 166                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 167
 168                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 169                 {
 170                         list($sClass, $sType) = explode(' ', trim($sLine));
 171                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 172                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 173
 174                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 175                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 176
 177                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 178                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 179                 }
 180                 fclose($hFile);
 181                 pgsqlRunScript($sScript);
 182         }
 183
 184         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 185         {
 186                 echo "Tables\n";
 187                 $bDidSomething = true;
 188                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 189
 190                 // re-run the functions
 191                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 192                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 193                 pgsqlRunScript($sTemplate);
 194         }
 195
 196         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 197         {
 198                 echo "Partitions\n";
 199                 $bDidSomething = true;
 200                 $oDB =& getDB();
 201                 $sSQL = 'select partition from country_name order by country_code';
 202                 $aPartitions = $oDB->getCol($sSQL);
 203                 if (PEAR::isError($aPartitions))
 204                 {
 205                         fail($aPartitions->getMessage());
 206                 }
 207                 $aPartitions[] = 0;
 208
 209                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 210                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 211                 foreach($aMatches as $aMatch)
 212                 {
 213                         $sResult = '';
 214                         foreach($aPartitions as $sPartitionName)
 215                         {
 216                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 217                         }
 218                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 219                 }
 220
 221                 pgsqlRunScript($sTemplate);
 222         }
 223
 224         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 225         {
 226                 $bDidSomething = true;
 227                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 228                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 229                 if (file_exists($sWikiArticlesFile))
 230                 {
 231                         echo "Importing wikipedia articles...";
 232                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 233                         echo "...done\n";
 234                 }
 235                 else
 236                 {
 237                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 238                 }
 239                 if (file_exists($sWikiRedirectsFile))
 240                 {
 241                         echo "Importing wikipedia redirects...";
 242                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 243                         echo "...done\n";
 244                 }
 245                 else
 246                 {
 247                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 248                 }
 249         }
 250
 251
 252         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 253         {
 254                 echo "Load Data\n";
 255                 $bDidSomething = true;
 256
 257                 $oDB =& getDB();
 258                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 259                 echo '.';
 260                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 261                 echo '.';
 262                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 263                 echo '.';
 264                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 265                 echo '.';
 266                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 267                 echo '.';
 268                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 269                 echo '.';
 270                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 271                 echo '.';
 272                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 273                 echo '.';
 274                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 275                 echo '.';
 276
 277                 $sSQL = 'select partition from country_name order by country_code';
 278                 $aPartitions = $oDB->getCol($sSQL);
 279                 if (PEAR::isError($aPartitions))
 280                 {
 281                         fail($aPartitions->getMessage());
 282                 }
 283                 $aPartitions[] = 0;
 284                 foreach($aPartitions as $sPartition)
 285                 {
 286                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 287                         echo '.';
 288                 }
 289
 290                 // pre-create the word list
 291                 if (!$aCMDResult['disable-token-precalc'])
 292                 {
 293                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 294                         echo '.';
 295                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 296                         echo '.';
 297                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 298                         echo '.';
 299                 }
 300
 301                 $aDBInstances = array();
 302                 for($i = 0; $i < $iInstances; $i++)
 303                 {
 304                         $aDBInstances[$i] =& getDB(true);
 305                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 306                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 307                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 308                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 309                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 310                 }
 311                 $bAnyBusy = true;
 312                 while($bAnyBusy)
 313                 {
 314                         $bAnyBusy = false;
 315                         for($i = 0; $i < $iInstances; $i++)
 316                         {
 317                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 318                         }
 319                         sleep(1);
 320                         echo '.';
 321                 }
 322                 echo "\n";
 323                 echo "Reanalysing database...\n";
 324                 pgsqlRunScript('ANALYSE');
 325         }
 326
 327         if ($aCMDResult['create-roads'])
 328         {
 329                 $bDidSomething = true;
 330
 331                 $oDB =& getDB();
 332                 $aDBInstances = array();
 333                 for($i = 0; $i < $iInstances; $i++)
 334                 {
 335                         $aDBInstances[$i] =& getDB(true);
 336                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 337                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 338                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 339                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 340                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 341                 }
 342                 $bAnyBusy = true;
 343                 while($bAnyBusy)
 344                 {
 345                         $bAnyBusy = false;
 346                         for($i = 0; $i < $iInstances; $i++)
 347                         {
 348                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 349                         }
 350                         sleep(1);
 351                         echo '.';
 352                 }
 353                 echo "\n";
 354         }
 355
 356         if ($aCMDResult['import-tiger-data'])
 357         {
 358                 $bDidSomething = true;
 359
 360                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 361
 362                 $aDBInstances = array();
 363                 for($i = 0; $i < $iInstances; $i++)
 364                 {
 365                         $aDBInstances[$i] =& getDB(true);
 366                 }
 367
 368                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 369                 {
 370                         echo $sFile.': ';
 371                         $hFile = fopen($sFile, "r");
 372                         $sSQL = fgets($hFile, 100000);
 373                         $iLines = 0;
 374
 375                         while(true)
 376                         {
 377                                 for($i = 0; $i < $iInstances; $i++)
 378                                 {
 379                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 380                                         {
 381                                                 while(pg_get_result($aDBInstances[$i]->connection));
 382                                                 $sSQL = fgets($hFile, 100000);
 383                                                 if (!$sSQL) break 2;
 384                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 385                                                 $iLines++;
 386                                                 if ($iLines == 1000)
 387                                                 {
 388                                                         echo ".";
 389                                                         $iLines = 0;
 390                                                 }
 391                                         }
 392                                 }
 393                                 usleep(10);
 394                         }
 395
 396                         fclose($hFile);
 397
 398                         $bAnyBusy = true;
 399                         while($bAnyBusy)
 400                         {
 401                                 $bAnyBusy = false;
 402                                 for($i = 0; $i < $iInstances; $i++)
 403                                 {
 404                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 405                                 }
 406                                 usleep(10);
 407                         }
 408                         echo "\n";
 409                 }
 410
 411                 echo "Creating indexes\n";
 412                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 413         }
 414
 415         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 416         {
 417                 $bDidSomething = true;
 418                 $oDB =& getDB();
 419                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 420                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 421                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 422                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 423                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 424                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 425                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 426
 427                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 428                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 429                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 430                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 431         }
 432
 433         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 434         {
 435                 $bDidSomething = true;
 436                 $oDB =& getDB();
 437
 438                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 439                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 440                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 441
 442                 $sDate = $aCMDResult['osmosis-init-date'];
 443                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 444                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 445                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 446                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 447                 $sURL .= '&stream=minute';
 448                 echo "Getting state file: $sURL\n";
 449                 $sStateFile = file_get_contents($sURL);
 450                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 451                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 452                 echo "Updating DB status\n";
 453                 pg_query($oDB->connection, 'TRUNCATE import_status');
 454                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 455                 pg_query($oDB->connection, $sSQL);
 456
 457         }
 458
 459         if ($aCMDResult['index'] || $aCMDResult['all'])
 460         {
 461                 $bDidSomething = true;
 462                 $sOutputFile = '';
 463                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 464                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 465                 passthruCheckReturn($sBaseCmd.' -R 4');
 466                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 467                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 468                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 469                 passthruCheckReturn($sBaseCmd.' -r 26');
 470         }
 471
 472         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 473         {
 474                 echo "Search indices\n";
 475                 $bDidSomething = true;
 476                 $oDB =& getDB();
 477                 $sSQL = 'select partition from country_name order by country_code';
 478                 $aPartitions = $oDB->getCol($sSQL);
 479                 if (PEAR::isError($aPartitions))
 480                 {
 481                         fail($aPartitions->getMessage());
 482                 }
 483                 $aPartitions[] = 0;
 484
 485                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 486                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 487                 foreach($aMatches as $aMatch)
 488                 {
 489                         $sResult = '';
 490                         foreach($aPartitions as $sPartitionName)
 491                         {
 492                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 493                         }
 494                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 495                 }
 496
 497                 pgsqlRunScript($sTemplate);
 498         }
 499
 500         if (isset($aCMDResult['create-website']))
 501         {
 502                 $bDidSomething = true;
 503                 $sTargetDir = $aCMDResult['create-website'];
 504                 if (!is_dir($sTargetDir))
 505                 {
 506                         echo "You must create the website directory before calling this function.\n";
 507                         fail("Target directory does not exist.");
 508                 }
 509
 510                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 511                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 512                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 513                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 514                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 515                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 516                 echo "Symlinks created\n";
 517         }
 518
 519         if (!$bDidSomething)
 520         {
 521                 showUsage($aCMDOptions, true);
 522         }
 523
 524         function pgsqlRunScriptFile($sFilename)
 525         {
 526                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 527
 528                 // Convert database DSN to psql parameters
 529                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 530                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 531                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 532
 533                 $aDescriptors = array(
 534                         0 => array('pipe', 'r'),
 535                         1 => array('pipe', 'w'),
 536                         2 => array('file', '/dev/null', 'a')
 537                 );
 538                 $ahPipes = null;
 539                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 540                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 541
 542                 fclose($ahPipes[0]);
 543
 544                 // TODO: error checking
 545                 while(!feof($ahPipes[1]))
 546                 {
 547                         echo fread($ahPipes[1], 4096);
 548                 }
 549                 fclose($ahPipes[1]);
 550
 551                 proc_close($hProcess);
 552         }
 553
 554         function pgsqlRunScript($sScript)
 555         {
 556                 // Convert database DSN to psql parameters
 557                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 558                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 559                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 560                 $aDescriptors = array(
 561                         0 => array('pipe', 'r'),
 562                         1 => STDOUT,
 563                         2 => STDERR
 564                 );
 565                 $ahPipes = null;
 566                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 567                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 568
 569                 while(strlen($sScript))
 570                 {
 571                         $written = fwrite($ahPipes[0], $sScript);
 572                         $sScript = substr($sScript, $written);
 573                 }
 574                 fclose($ahPipes[0]);
 575                 proc_close($hProcess);
 576         }
 577
 578         function pgsqlRunRestoreData($sDumpFile)
 579         {
 580                 // Convert database DSN to psql parameters
 581                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 582                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 583                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 584
 585                 $aDescriptors = array(
 586                         0 => array('pipe', 'r'),
 587                         1 => array('pipe', 'w'),
 588                         2 => array('file', '/dev/null', 'a')
 589                 );
 590                 $ahPipes = null;
 591                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 592                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 593
 594                 fclose($ahPipes[0]);
 595
 596                 // TODO: error checking
 597                 while(!feof($ahPipes[1]))
 598                 {
 599                         echo fread($ahPipes[1], 4096);
 600                 }
 601                 fclose($ahPipes[1]);
 602
 603                 proc_close($hProcess);
 604         }
 605
 606         function pgsqlRunDropAndRestore($sDumpFile)
 607         {
 608                 // Convert database DSN to psql parameters
 609                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 610                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 611                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 612
 613                 $aDescriptors = array(
 614                         0 => array('pipe', 'r'),
 615                         1 => array('pipe', 'w'),
 616                         2 => array('file', '/dev/null', 'a')
 617                 );
 618                 $ahPipes = null;
 619                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 620                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 621
 622                 fclose($ahPipes[0]);
 623
 624                 // TODO: error checking
 625                 while(!feof($ahPipes[1]))
 626                 {
 627                         echo fread($ahPipes[1], 4096);
 628                 }
 629                 fclose($ahPipes[1]);
 630
 631                 proc_close($hProcess);
 632         }
 633
 634         function passthruCheckReturn($cmd)
 635         {
 636                 $result = -1;
 637                 passthru($cmd, $result);
 638                 if ($result != 0) fail('Error executing external command: '.$cmd);
 639         }