utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  48         {
  49                 if (!isset($aCMDResult['osm-file']))
  50                 {
  51                         fail('missing --osm-file for data import');
  52                 }
  53
  54                 if (!file_exists($aCMDResult['osm-file']))
  55                 {
  56                         fail('the path supplied to --osm-file does not exist');
  57                 }
  58
  59                 if (!is_readable($aCMDResult['osm-file']))
  60                 {
  61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  62                 }
  63         }
  64
  65
  66         // This is a pretty hard core default - the number of processors in the box - 1
  67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  68         if ($iInstances < 1)
  69         {
  70                 $iInstances = 1;
  71                 echo "WARNING: resetting threads to $iInstances\n";
  72         }
  73         if ($iInstances > getProcessorCount())
  74         {
  75                 $iInstances = getProcessorCount();
  76                 echo "WARNING: resetting threads to $iInstances\n";
  77         }
  78
  79         // Assume we can steal all the cache memory in the box (unless told otherwise)
  80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  81         if ($iCacheMemory > getTotalMemoryMB())
  82         {
  83                 $iCacheMemory = getCacheMemoryMB();
  84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  85         }
  86
  87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  88         {
  89                 $sBaseFile = basename($aCMDResult['osm-file']);
  90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  91                 {
  92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  93                         $iTime -= (60*60*24);
  94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  95                 }
  96         }
  97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  99
 100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
 101         {
 102                 echo "Create DB\n";
 103                 $bDidSomething = true;
 104                 $oDB =& DB::connect(CONST_Database_DSN, false);
 105                 if (!PEAR::isError($oDB))
 106                 {
 107                         fail('database already exists ('.CONST_Database_DSN.')');
 108                 }
 109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
 110         }
 111
 112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 113         {
 114                 echo "Setup DB\n";
 115                 $bDidSomething = true;
 116                 // TODO: path detection, detection memory, etc.
 117
 118                 $oDB =& getDB();
 119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                 } else {
 124                         pgsqlRunScript('CREATE EXTENSION hstore');
 125                 }
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 136         }
 137
 138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 139         {
 140                 echo "Import\n";
 141                 $bDidSomething = true;
 142
 143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 144                 if (!file_exists($osm2pgsql))
 145                 {
 146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 147                         fail("osm2pgsql not found in '$osm2pgsql'");
 148                 }
 149                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 150                 $osm2pgsql .= ' -C '.$iCacheMemory;
 151                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 152                 passthruCheckReturn($osm2pgsql);
 153
 154                 $oDB =& getDB();
 155                 $x = $oDB->getRow('select * from place limit 1');
 156                 if (PEAR::isError($x)) {
 157                         fail($x->getMessage());
 158                 }
 159                 if (!$x) fail('No Data');
 160         }
 161
 162         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 163         {
 164                 echo "Functions\n";
 165                 $bDidSomething = true;
 166                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 167                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 168                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 169                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 170                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 171                 pgsqlRunScript($sTemplate);
 172         }
 173
 174         if ($aCMDResult['create-minimal-tables'])
 175         {
 176                 echo "Minimal Tables\n";
 177                 $bDidSomething = true;
 178                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 179
 180                 $sScript = '';
 181
 182                 // Backstop the import process - easliest possible import id
 183                 $sScript .= "insert into import_npi_log values (18022);\n";
 184
 185                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 186                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 187
 188                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 189                 {
 190                         list($sClass, $sType) = explode(' ', trim($sLine));
 191                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 192                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 193
 194                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 195                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 196
 197                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 198                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 199                 }
 200                 fclose($hFile);
 201                 pgsqlRunScript($sScript);
 202         }
 203
 204         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 205         {
 206                 echo "Tables\n";
 207                 $bDidSomething = true;
 208                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 209
 210                 // re-run the functions
 211                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 212                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 213                 pgsqlRunScript($sTemplate);
 214         }
 215
 216         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 217         {
 218                 echo "Partitions\n";
 219                 $bDidSomething = true;
 220                 $oDB =& getDB();
 221                 $sSQL = 'select partition from country_name order by country_code';
 222                 $aPartitions = $oDB->getCol($sSQL);
 223                 if (PEAR::isError($aPartitions))
 224                 {
 225                         fail($aPartitions->getMessage());
 226                 }
 227                 $aPartitions[] = 0;
 228
 229                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 230                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 231                 foreach($aMatches as $aMatch)
 232                 {
 233                         $sResult = '';
 234                         foreach($aPartitions as $sPartitionName)
 235                         {
 236                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 237                         }
 238                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 239                 }
 240
 241                 pgsqlRunScript($sTemplate);
 242         }
 243
 244         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 245         {
 246                 $bDidSomething = true;
 247                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 248                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 249                 if (file_exists($sWikiArticlesFile))
 250                 {
 251                         echo "Importing wikipedia articles...";
 252                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 253                         echo "...done\n";
 254                 }
 255                 else
 256                 {
 257                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 258                 }
 259                 if (file_exists($sWikiRedirectsFile))
 260                 {
 261                         echo "Importing wikipedia redirects...";
 262                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 263                         echo "...done\n";
 264                 }
 265                 else
 266                 {
 267                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 268                 }
 269         }
 270
 271
 272         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 273         {
 274                 echo "Load Data\n";
 275                 $bDidSomething = true;
 276
 277                 $oDB =& getDB();
 278                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 279                 echo '.';
 280                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 281                 echo '.';
 282                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 283                 echo '.';
 284                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 285                 echo '.';
 286                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 287                 echo '.';
 288                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 289                 echo '.';
 290                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 291                 echo '.';
 292                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 293                 echo '.';
 294                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 295                 echo '.';
 296
 297                 $sSQL = 'select partition from country_name order by country_code';
 298                 $aPartitions = $oDB->getCol($sSQL);
 299                 if (PEAR::isError($aPartitions))
 300                 {
 301                         fail($aPartitions->getMessage());
 302                 }
 303                 $aPartitions[] = 0;
 304                 foreach($aPartitions as $sPartition)
 305                 {
 306                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 307                         echo '.';
 308                 }
 309
 310                 // pre-create the word list
 311                 if (!$aCMDResult['disable-token-precalc'])
 312                 {
 313                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 314                         echo '.';
 315                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 316                         echo '.';
 317                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 318                         echo '.';
 319                 }
 320
 321                 $aDBInstances = array();
 322                 for($i = 0; $i < $iInstances; $i++)
 323                 {
 324                         $aDBInstances[$i] =& getDB(true);
 325                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 326                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 327                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 328                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 329                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 330                 }
 331                 $bAnyBusy = true;
 332                 while($bAnyBusy)
 333                 {
 334                         $bAnyBusy = false;
 335                         for($i = 0; $i < $iInstances; $i++)
 336                         {
 337                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 338                         }
 339                         sleep(1);
 340                         echo '.';
 341                 }
 342                 echo "\n";
 343                 echo "Reanalysing database...\n";
 344                 pgsqlRunScript('ANALYSE');
 345         }
 346
 347         if ($aCMDResult['create-roads'])
 348         {
 349                 $bDidSomething = true;
 350
 351                 $oDB =& getDB();
 352                 $aDBInstances = array();
 353                 for($i = 0; $i < $iInstances; $i++)
 354                 {
 355                         $aDBInstances[$i] =& getDB(true);
 356                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 357                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 358                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 359                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 360                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 361                 }
 362                 $bAnyBusy = true;
 363                 while($bAnyBusy)
 364                 {
 365                         $bAnyBusy = false;
 366                         for($i = 0; $i < $iInstances; $i++)
 367                         {
 368                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 369                         }
 370                         sleep(1);
 371                         echo '.';
 372                 }
 373                 echo "\n";
 374         }
 375
 376         if ($aCMDResult['import-tiger-data'])
 377         {
 378                 $bDidSomething = true;
 379
 380                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 381
 382                 $aDBInstances = array();
 383                 for($i = 0; $i < $iInstances; $i++)
 384                 {
 385                         $aDBInstances[$i] =& getDB(true);
 386                 }
 387
 388                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 389                 {
 390                         echo $sFile.': ';
 391                         $hFile = fopen($sFile, "r");
 392                         $sSQL = fgets($hFile, 100000);
 393                         $iLines = 0;
 394
 395                         while(true)
 396                         {
 397                                 for($i = 0; $i < $iInstances; $i++)
 398                                 {
 399                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 400                                         {
 401                                                 while(pg_get_result($aDBInstances[$i]->connection));
 402                                                 $sSQL = fgets($hFile, 100000);
 403                                                 if (!$sSQL) break 2;
 404                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 405                                                 $iLines++;
 406                                                 if ($iLines == 1000)
 407                                                 {
 408                                                         echo ".";
 409                                                         $iLines = 0;
 410                                                 }
 411                                         }
 412                                 }
 413                                 usleep(10);
 414                         }
 415
 416                         fclose($hFile);
 417
 418                         $bAnyBusy = true;
 419                         while($bAnyBusy)
 420                         {
 421                                 $bAnyBusy = false;
 422                                 for($i = 0; $i < $iInstances; $i++)
 423                                 {
 424                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 425                                 }
 426                                 usleep(10);
 427                         }
 428                         echo "\n";
 429                 }
 430
 431                 echo "Creating indexes\n";
 432                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 433         }
 434
 435         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 436         {
 437                 $bDidSomething = true;
 438                 $oDB =& getDB();
 439                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 440                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 441                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 442                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 443                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 444                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 445                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 446
 447                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 448                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 449                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 450                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 451         }
 452
 453         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 454         {
 455                 $bDidSomething = true;
 456                 $oDB =& getDB();
 457
 458                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 459                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 460                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 461
 462                 $sDate = $aCMDResult['osmosis-init-date'];
 463                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 464                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 465                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 466                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 467                 $sURL .= '&stream=minute';
 468                 echo "Getting state file: $sURL\n";
 469                 $sStateFile = file_get_contents($sURL);
 470                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 471                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 472                 echo "Updating DB status\n";
 473                 pg_query($oDB->connection, 'TRUNCATE import_status');
 474                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 475                 pg_query($oDB->connection, $sSQL);
 476
 477         }
 478
 479         if ($aCMDResult['index'] || $aCMDResult['all'])
 480         {
 481                 $bDidSomething = true;
 482                 $sOutputFile = '';
 483                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 484                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 485                 passthruCheckReturn($sBaseCmd.' -R 4');
 486                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 487                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 488                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 489                 passthruCheckReturn($sBaseCmd.' -r 26');
 490         }
 491
 492         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 493         {
 494                 echo "Search indices\n";
 495                 $bDidSomething = true;
 496                 $oDB =& getDB();
 497                 $sSQL = 'select partition from country_name order by country_code';
 498                 $aPartitions = $oDB->getCol($sSQL);
 499                 if (PEAR::isError($aPartitions))
 500                 {
 501                         fail($aPartitions->getMessage());
 502                 }
 503                 $aPartitions[] = 0;
 504
 505                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 506                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 507                 foreach($aMatches as $aMatch)
 508                 {
 509                         $sResult = '';
 510                         foreach($aPartitions as $sPartitionName)
 511                         {
 512                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 513                         }
 514                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 515                 }
 516
 517                 pgsqlRunScript($sTemplate);
 518         }
 519
 520         if (isset($aCMDResult['create-website']))
 521         {
 522                 $bDidSomething = true;
 523                 $sTargetDir = $aCMDResult['create-website'];
 524                 if (!is_dir($sTargetDir))
 525                 {
 526                         echo "You must create the website directory before calling this function.\n";
 527                         fail("Target directory does not exist.");
 528                 }
 529
 530                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 531                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 532                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 533                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 534                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 535                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 536                 echo "Symlinks created\n";
 537         }
 538
 539         if (!$bDidSomething)
 540         {
 541                 showUsage($aCMDOptions, true);
 542         }
 543
 544         function pgsqlRunScriptFile($sFilename)
 545         {
 546                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 547
 548                 // Convert database DSN to psql parameters
 549                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 550                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 551                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 552
 553                 $aDescriptors = array(
 554                         0 => array('pipe', 'r'),
 555                         1 => array('pipe', 'w'),
 556                         2 => array('file', '/dev/null', 'a')
 557                 );
 558                 $ahPipes = null;
 559                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 560                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 561
 562                 fclose($ahPipes[0]);
 563
 564                 // TODO: error checking
 565                 while(!feof($ahPipes[1]))
 566                 {
 567                         echo fread($ahPipes[1], 4096);
 568                 }
 569                 fclose($ahPipes[1]);
 570
 571                 proc_close($hProcess);
 572         }
 573
 574         function pgsqlRunScript($sScript)
 575         {
 576                 // Convert database DSN to psql parameters
 577                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 578                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 579                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 580                 $aDescriptors = array(
 581                         0 => array('pipe', 'r'),
 582                         1 => STDOUT,
 583                         2 => STDERR
 584                 );
 585                 $ahPipes = null;
 586                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 587                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 588
 589                 while(strlen($sScript))
 590                 {
 591                         $written = fwrite($ahPipes[0], $sScript);
 592                         $sScript = substr($sScript, $written);
 593                 }
 594                 fclose($ahPipes[0]);
 595                 proc_close($hProcess);
 596         }
 597
 598         function pgsqlRunRestoreData($sDumpFile)
 599         {
 600                 // Convert database DSN to psql parameters
 601                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 602                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 603                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 604
 605                 $aDescriptors = array(
 606                         0 => array('pipe', 'r'),
 607                         1 => array('pipe', 'w'),
 608                         2 => array('file', '/dev/null', 'a')
 609                 );
 610                 $ahPipes = null;
 611                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 612                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 613
 614                 fclose($ahPipes[0]);
 615
 616                 // TODO: error checking
 617                 while(!feof($ahPipes[1]))
 618                 {
 619                         echo fread($ahPipes[1], 4096);
 620                 }
 621                 fclose($ahPipes[1]);
 622
 623                 proc_close($hProcess);
 624         }
 625
 626         function pgsqlRunDropAndRestore($sDumpFile)
 627         {
 628                 // Convert database DSN to psql parameters
 629                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 630                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 631                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 632
 633                 $aDescriptors = array(
 634                         0 => array('pipe', 'r'),
 635                         1 => array('pipe', 'w'),
 636                         2 => array('file', '/dev/null', 'a')
 637                 );
 638                 $ahPipes = null;
 639                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 640                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 641
 642                 fclose($ahPipes[0]);
 643
 644                 // TODO: error checking
 645                 while(!feof($ahPipes[1]))
 646                 {
 647                         echo fread($ahPipes[1], 4096);
 648                 }
 649                 fclose($ahPipes[1]);
 650
 651                 proc_close($hProcess);
 652         }
 653
 654         function passthruCheckReturn($cmd)
 655         {
 656                 $result = -1;
 657                 passthru($cmd, $result);
 658                 if ($result != 0) fail('Error executing external command: '.$cmd);
 659         }