utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  40         );
  41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  42
  43         $bDidSomething = false;
  44
  45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  47         {
  48                 if (!isset($aCMDResult['osm-file']))
  49                 {
  50                         fail('missing --osm-file for data import');
  51                 }
  52
  53                 if (!file_exists($aCMDResult['osm-file']))
  54                 {
  55                         fail('the path supplied to --osm-file does not exist');
  56                 }
  57
  58                 if (!is_readable($aCMDResult['osm-file']))
  59                 {
  60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  61                 }
  62         }
  63
  64
  65         // This is a pretty hard core default - the number of processors in the box - 1
  66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  67         if ($iInstances < 1)
  68         {
  69                 $iInstances = 1;
  70                 echo "WARNING: resetting threads to $iInstances\n";
  71         }
  72         if ($iInstances > getProcessorCount())
  73         {
  74                 $iInstances = getProcessorCount();
  75                 echo "WARNING: resetting threads to $iInstances\n";
  76         }
  77
  78         // Assume we can steal all the cache memory in the box (unless told otherwise)
  79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  80         if ($iCacheMemory > getTotalMemoryMB())
  81         {
  82                 $iCacheMemory = getCacheMemoryMB();
  83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  84         }
  85
  86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  88
  89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  90         {
  91                 echo "Create DB\n";
  92                 $bDidSomething = true;
  93                 $oDB =& DB::connect(CONST_Database_DSN, false);
  94                 if (!PEAR::isError($oDB))
  95                 {
  96                         fail('database already exists ('.CONST_Database_DSN.')');
  97                 }
  98                 passthruCheckReturn('createdb -E UTF-8 '.$aDSNInfo['database']);
  99         }
 100
 101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 102         {
 103                 echo "Setup DB\n";
 104                 $bDidSomething = true;
 105                 // TODO: path detection, detection memory, etc.
 106
 107                 $oDB =& getDB();
 108
 109                 $sVersionString = $oDB->getOne('select version()');
 110                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 111                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 112                 {
 113                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 114                         exit;
 115                 }
 116
 117                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 118                 $pgver = (float) CONST_Postgresql_Version;
 119                 if ($pgver < 9.1) {
 120                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 121                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 122                 } else {
 123                         pgsqlRunScript('CREATE EXTENSION hstore');
 124                 }
 125
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 128                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 129                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 130                 {
 131                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 132                         exit;
 133                 }
 134
 135                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 136                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 137                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 138                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 139                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 140                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 141                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 142                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 143                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 144         }
 145
 146         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 147         {
 148                 echo "Import\n";
 149                 $bDidSomething = true;
 150
 151                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 152                 if (!file_exists($osm2pgsql))
 153                 {
 154                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 155                         fail("osm2pgsql not found in '$osm2pgsql'");
 156                 }
 157                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 158                 $osm2pgsql .= ' -C '.$iCacheMemory;
 159                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 160                 passthruCheckReturn($osm2pgsql);
 161
 162                 $oDB =& getDB();
 163                 $x = $oDB->getRow('select * from place limit 1');
 164                 if (PEAR::isError($x)) {
 165                         fail($x->getMessage());
 166                 }
 167                 if (!$x) fail('No Data');
 168         }
 169
 170         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 171         {
 172                 echo "Functions\n";
 173                 $bDidSomething = true;
 174                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 175                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 176                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 177                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 178                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 179                 pgsqlRunScript($sTemplate);
 180         }
 181
 182         if ($aCMDResult['create-minimal-tables'])
 183         {
 184                 echo "Minimal Tables\n";
 185                 $bDidSomething = true;
 186                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 187
 188                 $sScript = '';
 189
 190                 // Backstop the import process - easliest possible import id
 191                 $sScript .= "insert into import_npi_log values (18022);\n";
 192
 193                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 194                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 195
 196                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 197                 {
 198                         list($sClass, $sType) = explode(' ', trim($sLine));
 199                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 200                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 201
 202                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 203                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 204
 205                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 206                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 207                 }
 208                 fclose($hFile);
 209                 pgsqlRunScript($sScript);
 210         }
 211
 212         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 213         {
 214                 echo "Tables\n";
 215                 $bDidSomething = true;
 216                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 217
 218                 // re-run the functions
 219                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 220                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 221                 pgsqlRunScript($sTemplate);
 222         }
 223
 224         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 225         {
 226                 echo "Partitions\n";
 227                 $bDidSomething = true;
 228                 $oDB =& getDB();
 229                 $sSQL = 'select partition from country_name order by country_code';
 230                 $aPartitions = $oDB->getCol($sSQL);
 231                 if (PEAR::isError($aPartitions))
 232                 {
 233                         fail($aPartitions->getMessage());
 234                 }
 235                 $aPartitions[] = 0;
 236
 237                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 238                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 239                 foreach($aMatches as $aMatch)
 240                 {
 241                         $sResult = '';
 242                         foreach($aPartitions as $sPartitionName)
 243                         {
 244                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 245                         }
 246                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 247                 }
 248
 249                 pgsqlRunScript($sTemplate);
 250         }
 251
 252         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 253         {
 254                 $bDidSomething = true;
 255                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 256                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 257                 if (file_exists($sWikiArticlesFile))
 258                 {
 259                         echo "Importing wikipedia articles...";
 260                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 261                         echo "...done\n";
 262                 }
 263                 else
 264                 {
 265                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 266                 }
 267                 if (file_exists($sWikiRedirectsFile))
 268                 {
 269                         echo "Importing wikipedia redirects...";
 270                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 271                         echo "...done\n";
 272                 }
 273                 else
 274                 {
 275                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 276                 }
 277         }
 278
 279
 280         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 281         {
 282                 echo "Drop old Data\n";
 283                 $bDidSomething = true;
 284
 285                 $oDB =& getDB();
 286                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 287                 echo '.';
 288                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 289                 echo '.';
 290                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 291                 echo '.';
 292                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 293                 echo '.';
 294                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 295                 echo '.';
 296                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 297                 echo '.';
 298                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 299                 echo '.';
 300                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 301                 echo '.';
 302                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 303                 echo '.';
 304
 305                 $sSQL = 'select partition from country_name order by country_code';
 306                 $aPartitions = $oDB->getCol($sSQL);
 307                 if (PEAR::isError($aPartitions))
 308                 {
 309                         fail($aPartitions->getMessage());
 310                 }
 311                 $aPartitions[] = 0;
 312                 foreach($aPartitions as $sPartition)
 313                 {
 314                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 315                         echo '.';
 316                 }
 317
 318                 // used by getorcreate_word_id to ignore frequent partial words
 319                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 320                 echo ".\n";
 321
 322                 // pre-create the word list
 323                 if (!$aCMDResult['disable-token-precalc'])
 324                 {
 325                         echo "Loading word list\n";
 326                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 327                 }
 328
 329                 echo "Load Data\n";
 330                 $aDBInstances = array();
 331                 for($i = 0; $i < $iInstances; $i++)
 332                 {
 333                         $aDBInstances[$i] =& getDB(true);
 334                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 335                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 336                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 337                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 338                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 339                 }
 340                 $bAnyBusy = true;
 341                 while($bAnyBusy)
 342                 {
 343                         $bAnyBusy = false;
 344                         for($i = 0; $i < $iInstances; $i++)
 345                         {
 346                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 347                         }
 348                         sleep(1);
 349                         echo '.';
 350                 }
 351                 echo "\n";
 352                 echo "Reanalysing database...\n";
 353                 pgsqlRunScript('ANALYSE');
 354         }
 355
 356         if ($aCMDResult['create-roads'])
 357         {
 358                 $bDidSomething = true;
 359
 360                 $oDB =& getDB();
 361                 $aDBInstances = array();
 362                 for($i = 0; $i < $iInstances; $i++)
 363                 {
 364                         $aDBInstances[$i] =& getDB(true);
 365                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 366                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 367                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 368                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 369                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 370                 }
 371                 $bAnyBusy = true;
 372                 while($bAnyBusy)
 373                 {
 374                         $bAnyBusy = false;
 375                         for($i = 0; $i < $iInstances; $i++)
 376                         {
 377                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 378                         }
 379                         sleep(1);
 380                         echo '.';
 381                 }
 382                 echo "\n";
 383         }
 384
 385         if ($aCMDResult['import-tiger-data'])
 386         {
 387                 $bDidSomething = true;
 388
 389                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 390
 391                 $aDBInstances = array();
 392                 for($i = 0; $i < $iInstances; $i++)
 393                 {
 394                         $aDBInstances[$i] =& getDB(true);
 395                 }
 396
 397                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 398                 {
 399                         echo $sFile.': ';
 400                         $hFile = fopen($sFile, "r");
 401                         $sSQL = fgets($hFile, 100000);
 402                         $iLines = 0;
 403
 404                         while(true)
 405                         {
 406                                 for($i = 0; $i < $iInstances; $i++)
 407                                 {
 408                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 409                                         {
 410                                                 while(pg_get_result($aDBInstances[$i]->connection));
 411                                                 $sSQL = fgets($hFile, 100000);
 412                                                 if (!$sSQL) break 2;
 413                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 414                                                 $iLines++;
 415                                                 if ($iLines == 1000)
 416                                                 {
 417                                                         echo ".";
 418                                                         $iLines = 0;
 419                                                 }
 420                                         }
 421                                 }
 422                                 usleep(10);
 423                         }
 424
 425                         fclose($hFile);
 426
 427                         $bAnyBusy = true;
 428                         while($bAnyBusy)
 429                         {
 430                                 $bAnyBusy = false;
 431                                 for($i = 0; $i < $iInstances; $i++)
 432                                 {
 433                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 434                                 }
 435                                 usleep(10);
 436                         }
 437                         echo "\n";
 438                 }
 439
 440                 echo "Creating indexes\n";
 441                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 442         }
 443
 444         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 445         {
 446                 $bDidSomething = true;
 447                 $oDB =& getDB();
 448                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 449                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 450                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 451                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 452                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 453                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 454                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 455
 456                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 457                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 458                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 459                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 460         }
 461
 462         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 463         {
 464                 $bDidSomething = true;
 465                 $oDB =& getDB();
 466
 467                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 468                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 469                 {
 470                         echo "settings/configuration.txt already exists\n";
 471                 }
 472                 else
 473                 {
 474                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 475                         // server layout changed afer license change, fix path to minutely diffs
 476                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 477                 }
 478
 479                 // Find the last node in the DB
 480                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
 481
 482                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 483                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
 484                 $sLastNodeXML = file_get_contents($sLastNodeURL);
 485                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 486                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 487
 488
 489                 // Search for the correct state file - uses file timestamps
 490                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
 491                 $sRep = file_get_contents($sRepURL);
 492                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 493                 $aPrevRepMatch = false;
 494                 foreach($aRepMatches as $aRepMatch)
 495                 {
 496                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 497                         $aPrevRepMatch = $aRepMatch;
 498                 }
 499                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 500
 501                 $sRepURL .= $aRepMatch[1];
 502                 $sRep = file_get_contents($sRepURL);
 503                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 504                 $aPrevRepMatch = false;
 505                 foreach($aRepMatches as $aRepMatch)
 506                 {
 507                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 508                         $aPrevRepMatch = $aRepMatch;
 509                 }
 510                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 511
 512                 $sRepURL .= $aRepMatch[1];
 513                 $sRep = file_get_contents($sRepURL);
 514                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 515                 $aPrevRepMatch = false;
 516                 foreach($aRepMatches as $aRepMatch)
 517                 {
 518                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 519                         $aPrevRepMatch = $aRepMatch;
 520                 }
 521                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 522
 523                 $sRepURL .= $aRepMatch[1].'.state.txt';
 524                 echo "Getting state file: $sRepURL\n";
 525                 $sStateFile = file_get_contents($sRepURL);
 526                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 527                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 528                 echo "Updating DB status\n";
 529                 pg_query($oDB->connection, 'TRUNCATE import_status');
 530                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 531                 pg_query($oDB->connection, $sSQL);
 532         }
 533
 534         if ($aCMDResult['index'] || $aCMDResult['all'])
 535         {
 536                 $bDidSomething = true;
 537                 $sOutputFile = '';
 538                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 539                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 540                 passthruCheckReturn($sBaseCmd.' -R 4');
 541                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 542                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 543                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 544                 passthruCheckReturn($sBaseCmd.' -r 26');
 545         }
 546
 547         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 548         {
 549                 echo "Search indices\n";
 550                 $bDidSomething = true;
 551                 $oDB =& getDB();
 552                 $sSQL = 'select partition from country_name order by country_code';
 553                 $aPartitions = $oDB->getCol($sSQL);
 554                 if (PEAR::isError($aPartitions))
 555                 {
 556                         fail($aPartitions->getMessage());
 557                 }
 558                 $aPartitions[] = 0;
 559
 560                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 561                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 562                 foreach($aMatches as $aMatch)
 563                 {
 564                         $sResult = '';
 565                         foreach($aPartitions as $sPartitionName)
 566                         {
 567                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 568                         }
 569                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 570                 }
 571
 572                 pgsqlRunScript($sTemplate);
 573         }
 574
 575         if (isset($aCMDResult['create-website']))
 576         {
 577                 $bDidSomething = true;
 578                 $sTargetDir = $aCMDResult['create-website'];
 579                 if (!is_dir($sTargetDir))
 580                 {
 581                         echo "You must create the website directory before calling this function.\n";
 582                         fail("Target directory does not exist.");
 583                 }
 584
 585                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 586                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 587                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 588                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 589                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 590                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 591                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 592                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 593                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 594                 echo "Symlinks created\n";
 595         }
 596
 597         if (!$bDidSomething)
 598         {
 599                 showUsage($aCMDOptions, true);
 600         }
 601
 602         function pgsqlRunScriptFile($sFilename)
 603         {
 604                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 605
 606                 // Convert database DSN to psql parameters
 607                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 608                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 609                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 610
 611                 $aDescriptors = array(
 612                         0 => array('pipe', 'r'),
 613                         1 => array('pipe', 'w'),
 614                         2 => array('file', '/dev/null', 'a')
 615                 );
 616                 $ahPipes = null;
 617                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 618                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 619
 620                 fclose($ahPipes[0]);
 621
 622                 // TODO: error checking
 623                 while(!feof($ahPipes[1]))
 624                 {
 625                         echo fread($ahPipes[1], 4096);
 626                 }
 627                 fclose($ahPipes[1]);
 628
 629                 proc_close($hProcess);
 630         }
 631
 632         function pgsqlRunScript($sScript)
 633         {
 634                 // Convert database DSN to psql parameters
 635                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 636                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 637                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 638                 $aDescriptors = array(
 639                         0 => array('pipe', 'r'),
 640                         1 => STDOUT,
 641                         2 => STDERR
 642                 );
 643                 $ahPipes = null;
 644                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 645                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 646
 647                 while(strlen($sScript))
 648                 {
 649                         $written = fwrite($ahPipes[0], $sScript);
 650                         $sScript = substr($sScript, $written);
 651                 }
 652                 fclose($ahPipes[0]);
 653                 proc_close($hProcess);
 654         }
 655
 656         function pgsqlRunRestoreData($sDumpFile)
 657         {
 658                 // Convert database DSN to psql parameters
 659                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 660                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 661                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 662
 663                 $aDescriptors = array(
 664                         0 => array('pipe', 'r'),
 665                         1 => array('pipe', 'w'),
 666                         2 => array('file', '/dev/null', 'a')
 667                 );
 668                 $ahPipes = null;
 669                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 670                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 671
 672                 fclose($ahPipes[0]);
 673
 674                 // TODO: error checking
 675                 while(!feof($ahPipes[1]))
 676                 {
 677                         echo fread($ahPipes[1], 4096);
 678                 }
 679                 fclose($ahPipes[1]);
 680
 681                 proc_close($hProcess);
 682         }
 683
 684         function pgsqlRunDropAndRestore($sDumpFile)
 685         {
 686                 // Convert database DSN to psql parameters
 687                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 688                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 689                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 690
 691                 $aDescriptors = array(
 692                         0 => array('pipe', 'r'),
 693                         1 => array('pipe', 'w'),
 694                         2 => array('file', '/dev/null', 'a')
 695                 );
 696                 $ahPipes = null;
 697                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 698                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 699
 700                 fclose($ahPipes[0]);
 701
 702                 // TODO: error checking
 703                 while(!feof($ahPipes[1]))
 704                 {
 705                         echo fread($ahPipes[1], 4096);
 706                 }
 707                 fclose($ahPipes[1]);
 708
 709                 proc_close($hProcess);
 710         }
 711
 712         function passthruCheckReturn($cmd)
 713         {
 714                 $result = -1;
 715                 passthru($cmd, $result);
 716                 if ($result != 0) fail('Error executing external command: '.$cmd);
 717         }