utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  48         {
  49                 if (!isset($aCMDResult['osm-file']))
  50                 {
  51                         fail('missing --osm-file for data import');
  52                 }
  53
  54                 if (!file_exists($aCMDResult['osm-file']))
  55                 {
  56                         fail('the path supplied to --osm-file does not exist');
  57                 }
  58
  59                 if (!is_readable($aCMDResult['osm-file']))
  60                 {
  61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  62                 }
  63         }
  64
  65
  66         // This is a pretty hard core default - the number of processors in the box - 1
  67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  68         if ($iInstances < 1)
  69         {
  70                 $iInstances = 1;
  71                 echo "WARNING: resetting threads to $iInstances\n";
  72         }
  73         if ($iInstances > getProcessorCount())
  74         {
  75                 $iInstances = getProcessorCount();
  76                 echo "WARNING: resetting threads to $iInstances\n";
  77         }
  78
  79         // Assume we can steal all the cache memory in the box (unless told otherwise)
  80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  81         if ($iCacheMemory > getTotalMemoryMB())
  82         {
  83                 $iCacheMemory = getCacheMemoryMB();
  84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  85         }
  86
  87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  88         {
  89                 $sBaseFile = basename($aCMDResult['osm-file']);
  90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  91                 {
  92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  93                         $iTime -= (60*60*24);
  94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  95                 }
  96         }
  97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  99
 100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
 101         {
 102                 echo "Create DB\n";
 103                 $bDidSomething = true;
 104                 $oDB =& DB::connect(CONST_Database_DSN, false);
 105                 if (!PEAR::isError($oDB))
 106                 {
 107                         fail('database already exists ('.CONST_Database_DSN.')');
 108                 }
 109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
 110         }
 111
 112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 113         {
 114                 echo "Setup DB\n";
 115                 $bDidSomething = true;
 116                 // TODO: path detection, detection memory, etc.
 117
 118                 $oDB =& getDB();
 119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                 } else {
 124                         pgsqlRunScript('CREATE EXTENSION hstore');
 125                 }
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 136         }
 137
 138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 139         {
 140                 echo "Import\n";
 141                 $bDidSomething = true;
 142
 143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 144                 if (!file_exists($osm2pgsql))
 145                 {
 146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 147                         fail("osm2pgsql not found in '$osm2pgsql'");
 148                 }
 149                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 150                 $osm2pgsql .= ' -C '.$iCacheMemory;
 151                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 152                 passthruCheckReturn($osm2pgsql);
 153
 154                 $oDB =& getDB();
 155                 $x = $oDB->getRow('select * from place limit 1');
 156                 if (PEAR::isError($x)) {
 157                         fail($x->getMessage());
 158                 }
 159                 if (!$x) fail('No Data');
 160         }
 161
 162         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 163         {
 164                 echo "Functions\n";
 165                 $bDidSomething = true;
 166                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 167                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 168                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 169                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 170                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 171                 pgsqlRunScript($sTemplate);
 172         }
 173
 174         if ($aCMDResult['create-minimal-tables'])
 175         {
 176                 echo "Minimal Tables\n";
 177                 $bDidSomething = true;
 178                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 179
 180                 $sScript = '';
 181
 182                 // Backstop the import process - easliest possible import id
 183                 $sScript .= "insert into import_npi_log values (18022);\n";
 184
 185                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 186                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 187
 188                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 189                 {
 190                         list($sClass, $sType) = explode(' ', trim($sLine));
 191                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 192                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 193
 194                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 195                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 196
 197                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 198                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 199                 }
 200                 fclose($hFile);
 201                 pgsqlRunScript($sScript);
 202         }
 203
 204         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 205         {
 206                 echo "Tables\n";
 207                 $bDidSomething = true;
 208                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 209
 210                 // re-run the functions
 211                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 212                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 213                 pgsqlRunScript($sTemplate);
 214         }
 215
 216         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 217         {
 218                 echo "Partitions\n";
 219                 $bDidSomething = true;
 220                 $oDB =& getDB();
 221                 $sSQL = 'select partition from country_name order by country_code';
 222                 $aPartitions = $oDB->getCol($sSQL);
 223                 if (PEAR::isError($aPartitions))
 224                 {
 225                         fail($aPartitions->getMessage());
 226                 }
 227                 $aPartitions[] = 0;
 228
 229                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 230                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 231                 foreach($aMatches as $aMatch)
 232                 {
 233                         $sResult = '';
 234                         foreach($aPartitions as $sPartitionName)
 235                         {
 236                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 237                         }
 238                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 239                 }
 240
 241                 pgsqlRunScript($sTemplate);
 242         }
 243
 244         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 245         {
 246                 $bDidSomething = true;
 247                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 248                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 249                 if (file_exists($sWikiArticlesFile))
 250                 {
 251                         echo "Importing wikipedia articles...";
 252                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 253                         echo "...done\n";
 254                 }
 255                 else
 256                 {
 257                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 258                 }
 259                 if (file_exists($sWikiRedirectsFile))
 260                 {
 261                         echo "Importing wikipedia redirects...";
 262                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 263                         echo "...done\n";
 264                 }
 265                 else
 266                 {
 267                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 268                 }
 269         }
 270
 271
 272         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 273         {
 274                 echo "Load Data\n";
 275                 $bDidSomething = true;
 276
 277                 $oDB =& getDB();
 278                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 279                 echo '.';
 280                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 281                 echo '.';
 282                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 283                 echo '.';
 284                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 285                 echo '.';
 286                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 287                 echo '.';
 288                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 289                 echo '.';
 290                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 291                 echo '.';
 292                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 293                 echo '.';
 294                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 295                 echo '.';
 296
 297                 $sSQL = 'select partition from country_name order by country_code';
 298                 $aPartitions = $oDB->getCol($sSQL);
 299                 if (PEAR::isError($aPartitions))
 300                 {
 301                         fail($aPartitions->getMessage());
 302                 }
 303                 $aPartitions[] = 0;
 304                 foreach($aPartitions as $sPartition)
 305                 {
 306                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 307                         echo '.';
 308                 }
 309
 310                 // pre-create the word list
 311                 if (!$aCMDResult['disable-token-precalc'])
 312                 {
 313                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 314                         echo '.';
 315                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 316                         echo '.';
 317                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 318                         echo '.';
 319                 }
 320
 321                 $aDBInstances = array();
 322                 for($i = 0; $i < $iInstances; $i++)
 323                 {
 324                         $aDBInstances[$i] =& getDB(true);
 325                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 326                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 327                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 328                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 329                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 330                 }
 331                 $bAnyBusy = true;
 332                 while($bAnyBusy)
 333                 {
 334                         $bAnyBusy = false;
 335                         for($i = 0; $i < $iInstances; $i++)
 336                         {
 337                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 338                         }
 339                         sleep(1);
 340                         echo '.';
 341                 }
 342                 echo "\n";
 343                 echo "Reanalysing database...\n";
 344                 pgsqlRunScript('ANALYSE');
 345         }
 346
 347         if ($aCMDResult['create-roads'])
 348         {
 349                 $bDidSomething = true;
 350
 351                 $oDB =& getDB();
 352                 $aDBInstances = array();
 353                 for($i = 0; $i < $iInstances; $i++)
 354                 {
 355                         $aDBInstances[$i] =& getDB(true);
 356                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 357                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 358                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 359                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 360                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 361                 }
 362                 $bAnyBusy = true;
 363                 while($bAnyBusy)
 364                 {
 365                         $bAnyBusy = false;
 366                         for($i = 0; $i < $iInstances; $i++)
 367                         {
 368                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 369                         }
 370                         sleep(1);
 371                         echo '.';
 372                 }
 373                 echo "\n";
 374         }
 375
 376         if ($aCMDResult['import-tiger-data'])
 377         {
 378                 $bDidSomething = true;
 379
 380                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 381
 382                 $aDBInstances = array();
 383                 for($i = 0; $i < $iInstances; $i++)
 384                 {
 385                         $aDBInstances[$i] =& getDB(true);
 386                 }
 387
 388                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 389                 {
 390                         echo $sFile.': ';
 391                         $hFile = fopen($sFile, "r");
 392                         $sSQL = fgets($hFile, 100000);
 393                         $iLines = 0;
 394
 395                         while(true)
 396                         {
 397                                 for($i = 0; $i < $iInstances; $i++)
 398                                 {
 399                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 400                                         {
 401                                                 while(pg_get_result($aDBInstances[$i]->connection));
 402                                                 $sSQL = fgets($hFile, 100000);
 403                                                 if (!$sSQL) break 2;
 404                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 405                                                 $iLines++;
 406                                                 if ($iLines == 1000)
 407                                                 {
 408                                                         echo ".";
 409                                                         $iLines = 0;
 410                                                 }
 411                                         }
 412                                 }
 413                                 usleep(10);
 414                         }
 415
 416                         fclose($hFile);
 417
 418                         $bAnyBusy = true;
 419                         while($bAnyBusy)
 420                         {
 421                                 $bAnyBusy = false;
 422                                 for($i = 0; $i < $iInstances; $i++)
 423                                 {
 424                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 425                                 }
 426                                 usleep(10);
 427                         }
 428                         echo "\n";
 429                 }
 430
 431                 echo "Creating indexes\n";
 432                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 433         }
 434
 435         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 436         {
 437                 $bDidSomething = true;
 438                 $oDB =& getDB();
 439                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 440                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 441                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 442                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 443                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 444                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 445                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 446
 447                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 448                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 449                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 450                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 451         }
 452
 453         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 454         {
 455                 $bDidSomething = true;
 456                 $oDB =& getDB();
 457
 458                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 459                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 460                 else
 461                 {
 462                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 463                         // server layout changed afer license change, fix path to minutely diffs
 464                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 465                 }
 466
 467                 $sDate = $aCMDResult['osmosis-init-date'];
 468                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 469                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 470                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 471                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 472                 $sURL .= '&stream=minute';
 473                 echo "Getting state file: $sURL\n";
 474                 $sStateFile = file_get_contents($sURL);
 475                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 476                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 477                 echo "Updating DB status\n";
 478                 pg_query($oDB->connection, 'TRUNCATE import_status');
 479                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 480                 pg_query($oDB->connection, $sSQL);
 481
 482         }
 483
 484         if ($aCMDResult['index'] || $aCMDResult['all'])
 485         {
 486                 $bDidSomething = true;
 487                 $sOutputFile = '';
 488                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 489                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 490                 passthruCheckReturn($sBaseCmd.' -R 4');
 491                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 492                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 493                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 494                 passthruCheckReturn($sBaseCmd.' -r 26');
 495         }
 496
 497         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 498         {
 499                 echo "Search indices\n";
 500                 $bDidSomething = true;
 501                 $oDB =& getDB();
 502                 $sSQL = 'select partition from country_name order by country_code';
 503                 $aPartitions = $oDB->getCol($sSQL);
 504                 if (PEAR::isError($aPartitions))
 505                 {
 506                         fail($aPartitions->getMessage());
 507                 }
 508                 $aPartitions[] = 0;
 509
 510                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 511                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 512                 foreach($aMatches as $aMatch)
 513                 {
 514                         $sResult = '';
 515                         foreach($aPartitions as $sPartitionName)
 516                         {
 517                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 518                         }
 519                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 520                 }
 521
 522                 pgsqlRunScript($sTemplate);
 523         }
 524
 525         if (isset($aCMDResult['create-website']))
 526         {
 527                 $bDidSomething = true;
 528                 $sTargetDir = $aCMDResult['create-website'];
 529                 if (!is_dir($sTargetDir))
 530                 {
 531                         echo "You must create the website directory before calling this function.\n";
 532                         fail("Target directory does not exist.");
 533                 }
 534
 535                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 536                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 537                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 538                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 539                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 540                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 541                 echo "Symlinks created\n";
 542         }
 543
 544         if (!$bDidSomething)
 545         {
 546                 showUsage($aCMDOptions, true);
 547         }
 548
 549         function pgsqlRunScriptFile($sFilename)
 550         {
 551                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 552
 553                 // Convert database DSN to psql parameters
 554                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 555                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 556                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 557
 558                 $aDescriptors = array(
 559                         0 => array('pipe', 'r'),
 560                         1 => array('pipe', 'w'),
 561                         2 => array('file', '/dev/null', 'a')
 562                 );
 563                 $ahPipes = null;
 564                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 565                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 566
 567                 fclose($ahPipes[0]);
 568
 569                 // TODO: error checking
 570                 while(!feof($ahPipes[1]))
 571                 {
 572                         echo fread($ahPipes[1], 4096);
 573                 }
 574                 fclose($ahPipes[1]);
 575
 576                 proc_close($hProcess);
 577         }
 578
 579         function pgsqlRunScript($sScript)
 580         {
 581                 // Convert database DSN to psql parameters
 582                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 583                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 584                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 585                 $aDescriptors = array(
 586                         0 => array('pipe', 'r'),
 587                         1 => STDOUT,
 588                         2 => STDERR
 589                 );
 590                 $ahPipes = null;
 591                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 592                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 593
 594                 while(strlen($sScript))
 595                 {
 596                         $written = fwrite($ahPipes[0], $sScript);
 597                         $sScript = substr($sScript, $written);
 598                 }
 599                 fclose($ahPipes[0]);
 600                 proc_close($hProcess);
 601         }
 602
 603         function pgsqlRunRestoreData($sDumpFile)
 604         {
 605                 // Convert database DSN to psql parameters
 606                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 607                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 608                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 609
 610                 $aDescriptors = array(
 611                         0 => array('pipe', 'r'),
 612                         1 => array('pipe', 'w'),
 613                         2 => array('file', '/dev/null', 'a')
 614                 );
 615                 $ahPipes = null;
 616                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 617                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 618
 619                 fclose($ahPipes[0]);
 620
 621                 // TODO: error checking
 622                 while(!feof($ahPipes[1]))
 623                 {
 624                         echo fread($ahPipes[1], 4096);
 625                 }
 626                 fclose($ahPipes[1]);
 627
 628                 proc_close($hProcess);
 629         }
 630
 631         function pgsqlRunDropAndRestore($sDumpFile)
 632         {
 633                 // Convert database DSN to psql parameters
 634                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 635                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 636                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 637
 638                 $aDescriptors = array(
 639                         0 => array('pipe', 'r'),
 640                         1 => array('pipe', 'w'),
 641                         2 => array('file', '/dev/null', 'a')
 642                 );
 643                 $ahPipes = null;
 644                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 645                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 646
 647                 fclose($ahPipes[0]);
 648
 649                 // TODO: error checking
 650                 while(!feof($ahPipes[1]))
 651                 {
 652                         echo fread($ahPipes[1], 4096);
 653                 }
 654                 fclose($ahPipes[1]);
 655
 656                 proc_close($hProcess);
 657         }
 658
 659         function passthruCheckReturn($cmd)
 660         {
 661                 $result = -1;
 662                 passthru($cmd, $result);
 663                 if ($result != 0) fail('Error executing external command: '.$cmd);
 664         }