utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  40         );
  41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  42
  43         $bDidSomething = false;
  44
  45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  47         {
  48                 if (!isset($aCMDResult['osm-file']))
  49                 {
  50                         fail('missing --osm-file for data import');
  51                 }
  52
  53                 if (!file_exists($aCMDResult['osm-file']))
  54                 {
  55                         fail('the path supplied to --osm-file does not exist');
  56                 }
  57
  58                 if (!is_readable($aCMDResult['osm-file']))
  59                 {
  60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  61                 }
  62         }
  63
  64
  65         // This is a pretty hard core default - the number of processors in the box - 1
  66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  67         if ($iInstances < 1)
  68         {
  69                 $iInstances = 1;
  70                 echo "WARNING: resetting threads to $iInstances\n";
  71         }
  72         if ($iInstances > getProcessorCount())
  73         {
  74                 $iInstances = getProcessorCount();
  75                 echo "WARNING: resetting threads to $iInstances\n";
  76         }
  77
  78         // Assume we can steal all the cache memory in the box (unless told otherwise)
  79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  80         if ($iCacheMemory > getTotalMemoryMB())
  81         {
  82                 $iCacheMemory = getCacheMemoryMB();
  83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  84         }
  85
  86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  88
  89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  90         {
  91                 echo "Create DB\n";
  92                 $bDidSomething = true;
  93                 $oDB =& DB::connect(CONST_Database_DSN, false);
  94                 if (!PEAR::isError($oDB))
  95                 {
  96                         fail('database already exists ('.CONST_Database_DSN.')');
  97                 }
  98                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
  99         }
 100
 101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 102         {
 103                 echo "Setup DB\n";
 104                 $bDidSomething = true;
 105                 // TODO: path detection, detection memory, etc.
 106
 107                 $oDB =& getDB();
 108
 109                 $sVersionString = $oDB->getOne('select version()');
 110                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 111                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 112                 {
 113                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 114                         exit;
 115                 }
 116
 117                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 118                 $pgver = (float) CONST_Postgresql_Version;
 119                 if ($pgver < 9.1) {
 120                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 121                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 122                 } else {
 123                         pgsqlRunScript('CREATE EXTENSION hstore');
 124                 }
 125
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 128                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 129                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 130                 {
 131                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 132                         exit;
 133                 }
 134
 135                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 136                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 137                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 138                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 139                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 140                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 141                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 142                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 143                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 144         }
 145
 146         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 147         {
 148                 echo "Import\n";
 149                 $bDidSomething = true;
 150
 151                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 152                 if (!file_exists($osm2pgsql))
 153                 {
 154                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 155                         fail("osm2pgsql not found in '$osm2pgsql'");
 156                 }
 157                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 158                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 159                 $osm2pgsql .= ' -C 16000';
 160                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 161                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 162                 passthruCheckReturn($osm2pgsql);
 163
 164                 $oDB =& getDB();
 165                 $x = $oDB->getRow('select * from place limit 1');
 166                 if (PEAR::isError($x)) {
 167                         fail($x->getMessage());
 168                 }
 169                 if (!$x) fail('No Data');
 170         }
 171
 172         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 173         {
 174                 echo "Functions\n";
 175                 $bDidSomething = true;
 176                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 177                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 178                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 179                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 180                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 181                 pgsqlRunScript($sTemplate);
 182         }
 183
 184         if ($aCMDResult['create-minimal-tables'])
 185         {
 186                 echo "Minimal Tables\n";
 187                 $bDidSomething = true;
 188                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 189
 190                 $sScript = '';
 191
 192                 // Backstop the import process - easliest possible import id
 193                 $sScript .= "insert into import_npi_log values (18022);\n";
 194
 195                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 196                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 197
 198                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 199                 {
 200                         list($sClass, $sType) = explode(' ', trim($sLine));
 201                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 202                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 203
 204                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 205                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 206
 207                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 208                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 209                 }
 210                 fclose($hFile);
 211                 pgsqlRunScript($sScript);
 212         }
 213
 214         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 215         {
 216                 echo "Tables\n";
 217                 $bDidSomething = true;
 218                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 219
 220                 // re-run the functions
 221                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 222                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 223                 pgsqlRunScript($sTemplate);
 224         }
 225
 226         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 227         {
 228                 echo "Partitions\n";
 229                 $bDidSomething = true;
 230                 $oDB =& getDB();
 231                 $sSQL = 'select partition from country_name order by country_code';
 232                 $aPartitions = $oDB->getCol($sSQL);
 233                 if (PEAR::isError($aPartitions))
 234                 {
 235                         fail($aPartitions->getMessage());
 236                 }
 237                 $aPartitions[] = 0;
 238
 239                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 240                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 241                 foreach($aMatches as $aMatch)
 242                 {
 243                         $sResult = '';
 244                         foreach($aPartitions as $sPartitionName)
 245                         {
 246                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 247                         }
 248                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 249                 }
 250
 251                 pgsqlRunScript($sTemplate);
 252         }
 253
 254         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 255         {
 256                 $bDidSomething = true;
 257                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 258                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 259                 if (file_exists($sWikiArticlesFile))
 260                 {
 261                         echo "Importing wikipedia articles...";
 262                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 263                         echo "...done\n";
 264                 }
 265                 else
 266                 {
 267                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 268                 }
 269                 if (file_exists($sWikiRedirectsFile))
 270                 {
 271                         echo "Importing wikipedia redirects...";
 272                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 273                         echo "...done\n";
 274                 }
 275                 else
 276                 {
 277                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 278                 }
 279         }
 280
 281
 282         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 283         {
 284                 echo "Drop old Data\n";
 285                 $bDidSomething = true;
 286
 287                 $oDB =& getDB();
 288                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 289                 echo '.';
 290                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 291                 echo '.';
 292                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 293                 echo '.';
 294                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 295                 echo '.';
 296                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 297                 echo '.';
 298                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 299                 echo '.';
 300                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 301                 echo '.';
 302                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 303                 echo '.';
 304                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 305                 echo '.';
 306
 307                 $sSQL = 'select partition from country_name order by country_code';
 308                 $aPartitions = $oDB->getCol($sSQL);
 309                 if (PEAR::isError($aPartitions))
 310                 {
 311                         fail($aPartitions->getMessage());
 312                 }
 313                 $aPartitions[] = 0;
 314                 foreach($aPartitions as $sPartition)
 315                 {
 316                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 317                         echo '.';
 318                 }
 319
 320                 // used by getorcreate_word_id to ignore frequent partial words
 321                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 322                 echo ".\n";
 323
 324                 // pre-create the word list
 325                 if (!$aCMDResult['disable-token-precalc'])
 326                 {
 327                         echo "Loading word list\n";
 328                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 329                 }
 330
 331                 echo "Load Data\n";
 332                 $aDBInstances = array();
 333                 for($i = 0; $i < $iInstances; $i++)
 334                 {
 335                         $aDBInstances[$i] =& getDB(true);
 336                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 337                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 338                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 339                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 340                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 341                 }
 342                 $bAnyBusy = true;
 343                 while($bAnyBusy)
 344                 {
 345                         $bAnyBusy = false;
 346                         for($i = 0; $i < $iInstances; $i++)
 347                         {
 348                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 349                         }
 350                         sleep(1);
 351                         echo '.';
 352                 }
 353                 echo "\n";
 354                 echo "Reanalysing database...\n";
 355                 pgsqlRunScript('ANALYSE');
 356         }
 357
 358         if ($aCMDResult['create-roads'])
 359         {
 360                 $bDidSomething = true;
 361
 362                 $oDB =& getDB();
 363                 $aDBInstances = array();
 364                 for($i = 0; $i < $iInstances; $i++)
 365                 {
 366                         $aDBInstances[$i] =& getDB(true);
 367                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 368                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 369                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 370                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 371                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 372                 }
 373                 $bAnyBusy = true;
 374                 while($bAnyBusy)
 375                 {
 376                         $bAnyBusy = false;
 377                         for($i = 0; $i < $iInstances; $i++)
 378                         {
 379                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 380                         }
 381                         sleep(1);
 382                         echo '.';
 383                 }
 384                 echo "\n";
 385         }
 386
 387         if ($aCMDResult['import-tiger-data'])
 388         {
 389                 $bDidSomething = true;
 390
 391                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 392
 393                 $aDBInstances = array();
 394                 for($i = 0; $i < $iInstances; $i++)
 395                 {
 396                         $aDBInstances[$i] =& getDB(true);
 397                 }
 398
 399                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 400                 {
 401                         echo $sFile.': ';
 402                         $hFile = fopen($sFile, "r");
 403                         $sSQL = fgets($hFile, 100000);
 404                         $iLines = 0;
 405
 406                         while(true)
 407                         {
 408                                 for($i = 0; $i < $iInstances; $i++)
 409                                 {
 410                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 411                                         {
 412                                                 while(pg_get_result($aDBInstances[$i]->connection));
 413                                                 $sSQL = fgets($hFile, 100000);
 414                                                 if (!$sSQL) break 2;
 415                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 416                                                 $iLines++;
 417                                                 if ($iLines == 1000)
 418                                                 {
 419                                                         echo ".";
 420                                                         $iLines = 0;
 421                                                 }
 422                                         }
 423                                 }
 424                                 usleep(10);
 425                         }
 426
 427                         fclose($hFile);
 428
 429                         $bAnyBusy = true;
 430                         while($bAnyBusy)
 431                         {
 432                                 $bAnyBusy = false;
 433                                 for($i = 0; $i < $iInstances; $i++)
 434                                 {
 435                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 436                                 }
 437                                 usleep(10);
 438                         }
 439                         echo "\n";
 440                 }
 441
 442                 echo "Creating indexes\n";
 443                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 444         }
 445
 446         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 447         {
 448                 $bDidSomething = true;
 449                 $oDB =& getDB();
 450                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 451                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 452                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 453                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 454                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 455                 $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";
 456                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 457
 458                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 459                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 460                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 461                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 462         }
 463
 464         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 465         {
 466                 $bDidSomething = true;
 467                 $oDB =& getDB();
 468
 469                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 470                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 471                 {
 472                         echo "settings/configuration.txt already exists\n";
 473                 }
 474                 else
 475                 {
 476                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 477                         // server layout changed afer license change, fix path to minutely diffs
 478                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 479                 }
 480
 481                 // Find the last node in the DB
 482                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
 483
 484                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 485                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
 486                 $sLastNodeXML = file_get_contents($sLastNodeURL);
 487                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 488                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 489
 490
 491                 // Search for the correct state file - uses file timestamps
 492                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
 493                 $sRep = file_get_contents($sRepURL);
 494                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 495                 $aPrevRepMatch = false;
 496                 foreach($aRepMatches as $aRepMatch)
 497                 {
 498                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 499                         $aPrevRepMatch = $aRepMatch;
 500                 }
 501                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 502
 503                 $sRepURL .= $aRepMatch[1];
 504                 $sRep = file_get_contents($sRepURL);
 505                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 506                 $aPrevRepMatch = false;
 507                 foreach($aRepMatches as $aRepMatch)
 508                 {
 509                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 510                         $aPrevRepMatch = $aRepMatch;
 511                 }
 512                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 513
 514                 $sRepURL .= $aRepMatch[1];
 515                 $sRep = file_get_contents($sRepURL);
 516                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 517                 $aPrevRepMatch = false;
 518                 foreach($aRepMatches as $aRepMatch)
 519                 {
 520                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 521                         $aPrevRepMatch = $aRepMatch;
 522                 }
 523                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 524
 525                 $sRepURL .= $aRepMatch[1].'.state.txt';
 526                 echo "Getting state file: $sRepURL\n";
 527                 $sStateFile = file_get_contents($sRepURL);
 528                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 529                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 530                 echo "Updating DB status\n";
 531                 pg_query($oDB->connection, 'TRUNCATE import_status');
 532                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 533                 pg_query($oDB->connection, $sSQL);
 534         }
 535
 536         if ($aCMDResult['index'] || $aCMDResult['all'])
 537         {
 538                 $bDidSomething = true;
 539                 $sOutputFile = '';
 540                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 541                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 542                 passthruCheckReturn($sBaseCmd.' -R 4');
 543                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 544                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 545                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 546                 passthruCheckReturn($sBaseCmd.' -r 26');
 547         }
 548
 549         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 550         {
 551                 echo "Search indices\n";
 552                 $bDidSomething = true;
 553                 $oDB =& getDB();
 554                 $sSQL = 'select partition from country_name order by country_code';
 555                 $aPartitions = $oDB->getCol($sSQL);
 556                 if (PEAR::isError($aPartitions))
 557                 {
 558                         fail($aPartitions->getMessage());
 559                 }
 560                 $aPartitions[] = 0;
 561
 562                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 563                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 564                 foreach($aMatches as $aMatch)
 565                 {
 566                         $sResult = '';
 567                         foreach($aPartitions as $sPartitionName)
 568                         {
 569                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 570                         }
 571                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 572                 }
 573
 574                 pgsqlRunScript($sTemplate);
 575         }
 576
 577         if (isset($aCMDResult['create-website']))
 578         {
 579                 $bDidSomething = true;
 580                 $sTargetDir = $aCMDResult['create-website'];
 581                 if (!is_dir($sTargetDir))
 582                 {
 583                         echo "You must create the website directory before calling this function.\n";
 584                         fail("Target directory does not exist.");
 585                 }
 586
 587                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 588                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 589                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 590                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 591                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 592                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 593                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 594                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 595                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 596                 echo "Symlinks created\n";
 597         }
 598
 599         if (!$bDidSomething)
 600         {
 601                 showUsage($aCMDOptions, true);
 602         }
 603
 604         function pgsqlRunScriptFile($sFilename)
 605         {
 606                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 607
 608                 // Convert database DSN to psql parameters
 609                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 610                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 611                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 612
 613                 $aDescriptors = array(
 614                         0 => array('pipe', 'r'),
 615                         1 => array('pipe', 'w'),
 616                         2 => array('file', '/dev/null', 'a')
 617                 );
 618                 $ahPipes = null;
 619                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 620                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 621
 622                 fclose($ahPipes[0]);
 623
 624                 // TODO: error checking
 625                 while(!feof($ahPipes[1]))
 626                 {
 627                         echo fread($ahPipes[1], 4096);
 628                 }
 629                 fclose($ahPipes[1]);
 630
 631                 proc_close($hProcess);
 632         }
 633
 634         function pgsqlRunScript($sScript)
 635         {
 636                 // Convert database DSN to psql parameters
 637                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 638                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 639                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 640                 $aDescriptors = array(
 641                         0 => array('pipe', 'r'),
 642                         1 => STDOUT,
 643                         2 => STDERR
 644                 );
 645                 $ahPipes = null;
 646                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 647                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 648
 649                 while(strlen($sScript))
 650                 {
 651                         $written = fwrite($ahPipes[0], $sScript);
 652                         $sScript = substr($sScript, $written);
 653                 }
 654                 fclose($ahPipes[0]);
 655                 proc_close($hProcess);
 656         }
 657
 658         function pgsqlRunRestoreData($sDumpFile)
 659         {
 660                 // Convert database DSN to psql parameters
 661                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 662                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 663                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 664
 665                 $aDescriptors = array(
 666                         0 => array('pipe', 'r'),
 667                         1 => array('pipe', 'w'),
 668                         2 => array('file', '/dev/null', 'a')
 669                 );
 670                 $ahPipes = null;
 671                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 672                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 673
 674                 fclose($ahPipes[0]);
 675
 676                 // TODO: error checking
 677                 while(!feof($ahPipes[1]))
 678                 {
 679                         echo fread($ahPipes[1], 4096);
 680                 }
 681                 fclose($ahPipes[1]);
 682
 683                 proc_close($hProcess);
 684         }
 685
 686         function pgsqlRunDropAndRestore($sDumpFile)
 687         {
 688                 // Convert database DSN to psql parameters
 689                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 690                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 691                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 692
 693                 $aDescriptors = array(
 694                         0 => array('pipe', 'r'),
 695                         1 => array('pipe', 'w'),
 696                         2 => array('file', '/dev/null', 'a')
 697                 );
 698                 $ahPipes = null;
 699                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 700                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 701
 702                 fclose($ahPipes[0]);
 703
 704                 // TODO: error checking
 705                 while(!feof($ahPipes[1]))
 706                 {
 707                         echo fread($ahPipes[1], 4096);
 708                 }
 709                 fclose($ahPipes[1]);
 710
 711                 proc_close($hProcess);
 712         }
 713
 714         function passthruCheckReturn($cmd)
 715         {
 716                 $result = -1;
 717                 passthru($cmd, $result);
 718                 if ($result != 0) fail('Error executing external command: '.$cmd);
 719         }