utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  48         {
  49                 if (!isset($aCMDResult['osm-file']))
  50                 {
  51                         fail('missing --osm-file for data import');
  52                 }
  53
  54                 if (!file_exists($aCMDResult['osm-file']))
  55                 {
  56                         fail('the path supplied to --osm-file does not exist');
  57                 }
  58
  59                 if (!is_readable($aCMDResult['osm-file']))
  60                 {
  61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  62                 }
  63         }
  64
  65
  66         // This is a pretty hard core default - the number of processors in the box - 1
  67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  68         if ($iInstances < 1)
  69         {
  70                 $iInstances = 1;
  71                 echo "WARNING: resetting threads to $iInstances\n";
  72         }
  73         if ($iInstances > getProcessorCount())
  74         {
  75                 $iInstances = getProcessorCount();
  76                 echo "WARNING: resetting threads to $iInstances\n";
  77         }
  78
  79         // Assume we can steal all the cache memory in the box (unless told otherwise)
  80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  81         if ($iCacheMemory > getTotalMemoryMB())
  82         {
  83                 $iCacheMemory = getCacheMemoryMB();
  84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  85         }
  86
  87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  88         {
  89                 $sBaseFile = basename($aCMDResult['osm-file']);
  90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  91                 {
  92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  93                         $iTime -= (60*60*24);
  94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  95                 }
  96         }
  97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  99
 100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
 101         {
 102                 echo "Create DB\n";
 103                 $bDidSomething = true;
 104                 $oDB =& DB::connect(CONST_Database_DSN, false);
 105                 if (!PEAR::isError($oDB))
 106                 {
 107                         fail('database already exists ('.CONST_Database_DSN.')');
 108                 }
 109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
 110         }
 111
 112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 113         {
 114                 echo "Setup DB\n";
 115                 $bDidSomething = true;
 116                 // TODO: path detection, detection memory, etc.
 117
 118                 $oDB =& getDB();
 119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                 } else {
 124                         pgsqlRunScript('CREATE EXTENSION hstore');
 125                 }
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 136         }
 137
 138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 139         {
 140                 echo "Import\n";
 141                 $bDidSomething = true;
 142
 143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 144                 if (!file_exists($osm2pgsql))
 145                 {
 146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 147                         fail("osm2pgsql not found in '$osm2pgsql'");
 148                 }
 149                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 150                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 151                 $osm2pgsql .= ' -C 16000';
 152                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 153                 passthruCheckReturn($osm2pgsql);
 154
 155                 $oDB =& getDB();
 156                 $x = $oDB->getRow('select * from place limit 1');
 157                 if (PEAR::isError($x)) {
 158                         fail($x->getMessage());
 159                 }
 160                 if (!$x) fail('No Data');
 161         }
 162
 163         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 164         {
 165                 echo "Functions\n";
 166                 $bDidSomething = true;
 167                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 168                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 169                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 170                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 171                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 172                 pgsqlRunScript($sTemplate);
 173         }
 174
 175         if ($aCMDResult['create-minimal-tables'])
 176         {
 177                 echo "Minimal Tables\n";
 178                 $bDidSomething = true;
 179                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 180
 181                 $sScript = '';
 182
 183                 // Backstop the import process - easliest possible import id
 184                 $sScript .= "insert into import_npi_log values (18022);\n";
 185
 186                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 187                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 188
 189                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 190                 {
 191                         list($sClass, $sType) = explode(' ', trim($sLine));
 192                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 193                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 194
 195                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 196                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 197
 198                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 199                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 200                 }
 201                 fclose($hFile);
 202                 pgsqlRunScript($sScript);
 203         }
 204
 205         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 206         {
 207                 echo "Tables\n";
 208                 $bDidSomething = true;
 209                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 210
 211                 // re-run the functions
 212                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 213                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 214                 pgsqlRunScript($sTemplate);
 215         }
 216
 217         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 218         {
 219                 echo "Partitions\n";
 220                 $bDidSomething = true;
 221                 $oDB =& getDB();
 222                 $sSQL = 'select partition from country_name order by country_code';
 223                 $aPartitions = $oDB->getCol($sSQL);
 224                 if (PEAR::isError($aPartitions))
 225                 {
 226                         fail($aPartitions->getMessage());
 227                 }
 228                 $aPartitions[] = 0;
 229
 230                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 231                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 232                 foreach($aMatches as $aMatch)
 233                 {
 234                         $sResult = '';
 235                         foreach($aPartitions as $sPartitionName)
 236                         {
 237                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 238                         }
 239                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 240                 }
 241
 242                 pgsqlRunScript($sTemplate);
 243         }
 244
 245         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 246         {
 247                 $bDidSomething = true;
 248                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 249                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 250                 if (file_exists($sWikiArticlesFile))
 251                 {
 252                         echo "Importing wikipedia articles...";
 253                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 254                         echo "...done\n";
 255                 }
 256                 else
 257                 {
 258                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 259                 }
 260                 if (file_exists($sWikiRedirectsFile))
 261                 {
 262                         echo "Importing wikipedia redirects...";
 263                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 264                         echo "...done\n";
 265                 }
 266                 else
 267                 {
 268                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 269                 }
 270         }
 271
 272
 273         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 274         {
 275                 echo "Drop old Data\n";
 276                 $bDidSomething = true;
 277
 278                 $oDB =& getDB();
 279                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 280                 echo '.';
 281                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 282                 echo '.';
 283                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 284                 echo '.';
 285                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 286                 echo '.';
 287                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 288                 echo '.';
 289                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 290                 echo '.';
 291                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 292                 echo '.';
 293                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 294                 echo '.';
 295                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 296                 echo '.';
 297
 298                 $sSQL = 'select partition from country_name order by country_code';
 299                 $aPartitions = $oDB->getCol($sSQL);
 300                 if (PEAR::isError($aPartitions))
 301                 {
 302                         fail($aPartitions->getMessage());
 303                 }
 304                 $aPartitions[] = 0;
 305                 foreach($aPartitions as $sPartition)
 306                 {
 307                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 308                         echo '.';
 309                 }
 310
 311                 // used by getorcreate_word_id to ignore frequent partial words
 312                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 313                 echo ".\n";
 314
 315                 // pre-create the word list
 316                 if (!$aCMDResult['disable-token-precalc'])
 317                 {
 318                         echo "Loading word list\n";
 319                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 320                 }
 321
 322                 echo "Load Data\n";
 323                 $aDBInstances = array();
 324                 for($i = 0; $i < $iInstances; $i++)
 325                 {
 326                         $aDBInstances[$i] =& getDB(true);
 327                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 328                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 329                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 330                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 331                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 332                 }
 333                 $bAnyBusy = true;
 334                 while($bAnyBusy)
 335                 {
 336                         $bAnyBusy = false;
 337                         for($i = 0; $i < $iInstances; $i++)
 338                         {
 339                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 340                         }
 341                         sleep(1);
 342                         echo '.';
 343                 }
 344                 echo "\n";
 345                 echo "Reanalysing database...\n";
 346                 pgsqlRunScript('ANALYSE');
 347         }
 348
 349         if ($aCMDResult['create-roads'])
 350         {
 351                 $bDidSomething = true;
 352
 353                 $oDB =& getDB();
 354                 $aDBInstances = array();
 355                 for($i = 0; $i < $iInstances; $i++)
 356                 {
 357                         $aDBInstances[$i] =& getDB(true);
 358                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 359                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 360                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 361                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 362                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 363                 }
 364                 $bAnyBusy = true;
 365                 while($bAnyBusy)
 366                 {
 367                         $bAnyBusy = false;
 368                         for($i = 0; $i < $iInstances; $i++)
 369                         {
 370                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 371                         }
 372                         sleep(1);
 373                         echo '.';
 374                 }
 375                 echo "\n";
 376         }
 377
 378         if ($aCMDResult['import-tiger-data'])
 379         {
 380                 $bDidSomething = true;
 381
 382                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 383
 384                 $aDBInstances = array();
 385                 for($i = 0; $i < $iInstances; $i++)
 386                 {
 387                         $aDBInstances[$i] =& getDB(true);
 388                 }
 389
 390                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 391                 {
 392                         echo $sFile.': ';
 393                         $hFile = fopen($sFile, "r");
 394                         $sSQL = fgets($hFile, 100000);
 395                         $iLines = 0;
 396
 397                         while(true)
 398                         {
 399                                 for($i = 0; $i < $iInstances; $i++)
 400                                 {
 401                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 402                                         {
 403                                                 while(pg_get_result($aDBInstances[$i]->connection));
 404                                                 $sSQL = fgets($hFile, 100000);
 405                                                 if (!$sSQL) break 2;
 406                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 407                                                 $iLines++;
 408                                                 if ($iLines == 1000)
 409                                                 {
 410                                                         echo ".";
 411                                                         $iLines = 0;
 412                                                 }
 413                                         }
 414                                 }
 415                                 usleep(10);
 416                         }
 417
 418                         fclose($hFile);
 419
 420                         $bAnyBusy = true;
 421                         while($bAnyBusy)
 422                         {
 423                                 $bAnyBusy = false;
 424                                 for($i = 0; $i < $iInstances; $i++)
 425                                 {
 426                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 427                                 }
 428                                 usleep(10);
 429                         }
 430                         echo "\n";
 431                 }
 432
 433                 echo "Creating indexes\n";
 434                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 435         }
 436
 437         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 438         {
 439                 $bDidSomething = true;
 440                 $oDB =& getDB();
 441                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 442                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 443                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 444                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 445                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 446                 $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";
 447                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 448
 449                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 450                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 451                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 452                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 453         }
 454
 455         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 456         {
 457                 $bDidSomething = true;
 458                 $oDB =& getDB();
 459
 460                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 461                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 462                 else
 463                 {
 464                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 465                         // server layout changed afer license change, fix path to minutely diffs
 466                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 467                 }
 468
 469                 $sDate = $aCMDResult['osmosis-init-date'];
 470                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 471                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 472                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 473                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 474                 $sURL .= '&stream=minute';
 475                 echo "Getting state file: $sURL\n";
 476                 $sStateFile = file_get_contents($sURL);
 477                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 478                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 479                 echo "Updating DB status\n";
 480                 pg_query($oDB->connection, 'TRUNCATE import_status');
 481                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 482                 pg_query($oDB->connection, $sSQL);
 483
 484         }
 485
 486         if ($aCMDResult['index'] || $aCMDResult['all'])
 487         {
 488                 $bDidSomething = true;
 489                 $sOutputFile = '';
 490                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 491                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 492                 passthruCheckReturn($sBaseCmd.' -R 4');
 493                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 494                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 495                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 496                 passthruCheckReturn($sBaseCmd.' -r 26');
 497         }
 498
 499         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 500         {
 501                 echo "Search indices\n";
 502                 $bDidSomething = true;
 503                 $oDB =& getDB();
 504                 $sSQL = 'select partition from country_name order by country_code';
 505                 $aPartitions = $oDB->getCol($sSQL);
 506                 if (PEAR::isError($aPartitions))
 507                 {
 508                         fail($aPartitions->getMessage());
 509                 }
 510                 $aPartitions[] = 0;
 511
 512                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 513                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 514                 foreach($aMatches as $aMatch)
 515                 {
 516                         $sResult = '';
 517                         foreach($aPartitions as $sPartitionName)
 518                         {
 519                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 520                         }
 521                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 522                 }
 523
 524                 pgsqlRunScript($sTemplate);
 525         }
 526
 527         if (isset($aCMDResult['create-website']))
 528         {
 529                 $bDidSomething = true;
 530                 $sTargetDir = $aCMDResult['create-website'];
 531                 if (!is_dir($sTargetDir))
 532                 {
 533                         echo "You must create the website directory before calling this function.\n";
 534                         fail("Target directory does not exist.");
 535                 }
 536
 537                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 538                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 539                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 540                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 541                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 542                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 543                 echo "Symlinks created\n";
 544         }
 545
 546         if (!$bDidSomething)
 547         {
 548                 showUsage($aCMDOptions, true);
 549         }
 550
 551         function pgsqlRunScriptFile($sFilename)
 552         {
 553                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 554
 555                 // Convert database DSN to psql parameters
 556                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 557                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 558                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 559
 560                 $aDescriptors = array(
 561                         0 => array('pipe', 'r'),
 562                         1 => array('pipe', 'w'),
 563                         2 => array('file', '/dev/null', 'a')
 564                 );
 565                 $ahPipes = null;
 566                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 567                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 568
 569                 fclose($ahPipes[0]);
 570
 571                 // TODO: error checking
 572                 while(!feof($ahPipes[1]))
 573                 {
 574                         echo fread($ahPipes[1], 4096);
 575                 }
 576                 fclose($ahPipes[1]);
 577
 578                 proc_close($hProcess);
 579         }
 580
 581         function pgsqlRunScript($sScript)
 582         {
 583                 // Convert database DSN to psql parameters
 584                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 585                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 586                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 587                 $aDescriptors = array(
 588                         0 => array('pipe', 'r'),
 589                         1 => STDOUT,
 590                         2 => STDERR
 591                 );
 592                 $ahPipes = null;
 593                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 594                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 595
 596                 while(strlen($sScript))
 597                 {
 598                         $written = fwrite($ahPipes[0], $sScript);
 599                         $sScript = substr($sScript, $written);
 600                 }
 601                 fclose($ahPipes[0]);
 602                 proc_close($hProcess);
 603         }
 604
 605         function pgsqlRunRestoreData($sDumpFile)
 606         {
 607                 // Convert database DSN to psql parameters
 608                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 609                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 610                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 611
 612                 $aDescriptors = array(
 613                         0 => array('pipe', 'r'),
 614                         1 => array('pipe', 'w'),
 615                         2 => array('file', '/dev/null', 'a')
 616                 );
 617                 $ahPipes = null;
 618                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 619                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 620
 621                 fclose($ahPipes[0]);
 622
 623                 // TODO: error checking
 624                 while(!feof($ahPipes[1]))
 625                 {
 626                         echo fread($ahPipes[1], 4096);
 627                 }
 628                 fclose($ahPipes[1]);
 629
 630                 proc_close($hProcess);
 631         }
 632
 633         function pgsqlRunDropAndRestore($sDumpFile)
 634         {
 635                 // Convert database DSN to psql parameters
 636                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 637                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 638                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 639
 640                 $aDescriptors = array(
 641                         0 => array('pipe', 'r'),
 642                         1 => array('pipe', 'w'),
 643                         2 => array('file', '/dev/null', 'a')
 644                 );
 645                 $ahPipes = null;
 646                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 647                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 648
 649                 fclose($ahPipes[0]);
 650
 651                 // TODO: error checking
 652                 while(!feof($ahPipes[1]))
 653                 {
 654                         echo fread($ahPipes[1], 4096);
 655                 }
 656                 fclose($ahPipes[1]);
 657
 658                 proc_close($hProcess);
 659         }
 660
 661         function passthruCheckReturn($cmd)
 662         {
 663                 $result = -1;
 664                 passthru($cmd, $result);
 665                 if ($result != 0) fail('Error executing external command: '.$cmd);
 666         }