From c036480ce247bed75a382a230e3a109d60eb19bd Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Wed, 29 Aug 2018 21:31:19 +0200 Subject: [PATCH 1/1] first draft of setupClass --- CMakeLists.txt | 1 + lib/setup_functions.php | 1042 +++------------------------------------ utils/setup.php | 234 +++++---- utils/setupClass.php | 823 +++++++++++++++++++++++++++++++ 4 files changed, 1036 insertions(+), 1064 deletions(-) mode change 100644 => 100755 CMakeLists.txt create mode 100755 utils/setupClass.php diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100644 new mode 100755 index af6b9a92..5812883d --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ set(CUSTOMFILES utils/query.php utils/server_compare.php utils/setup.php + utils/setupClass.php utils/specialphrases.php utils/update.php utils/warm.php diff --git a/lib/setup_functions.php b/lib/setup_functions.php index b6d927ab..da66fc7d 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -1,965 +1,77 @@ - $aDSNInfo['password']), $_ENV); - } - - $result = runWithEnv($sCreateDBCmd, $aProcEnv); - if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); - } - - if ($aCMDResult['setup-db'] || $aCMDResult['all']) { - info('Setup DB'); - $bDidSomething = true; - - $oDB =& getDB(); - - $fPostgresVersion = getPostgresVersion($oDB); - echo 'Postgres version found: '.$fPostgresVersion."\n"; - - if ($fPostgresVersion < 9.1) { - fail('Minimum supported version of Postgresql is 9.1.'); - } - - pgsqlRunScript($aCMDResult, 'CREATE EXTENSION IF NOT EXISTS hstore'); - pgsqlRunScript($aCMDResult, 'CREATE EXTENSION IF NOT EXISTS postgis'); - - // For extratags and namedetails the hstore_to_json converter is - // needed which is only available from Postgresql 9.3+. For older - // versions add a dummy function that returns nothing. - $iNumFunc = chksql($oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); - - if ($iNumFunc == 0) { - pgsqlRunScript($aCMDResult, "create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); - warn('Postgresql is too old. extratags and namedetails API not available.'); - } - - $fPostgisVersion = getPostgisVersion($oDB); - echo 'Postgis version found: '.$fPostgisVersion."\n"; - - if ($fPostgisVersion < 2.1) { - // Functions were renamed in 2.1 and throw an annoying deprecation warning - pgsqlRunScript($aCMDResult, 'ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); - pgsqlRunScript($aCMDResult, 'ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); - } - if ($fPostgisVersion < 2.2) { - pgsqlRunScript($aCMDResult, 'ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); - } - - $i = chksql($oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); - if ($i == 0) { - echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; - echo "\n createuser ".CONST_Database_Web_User."\n\n"; - exit(1); - } - - if (!checkModulePresence()) { - fail('error loading nominatim.so module'); - } - - if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { - echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; - exit(1); - } - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_name.sql'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_naturalearthdata.sql'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_osm_grid.sql.gz'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/gb_postcode_table.sql'); - - - if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/gb_postcode_data.sql.gz'); - } else { - warn('external UK postcode table not found.'); - } - - if (CONST_Use_Extra_US_Postcodes) { - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/us_postcode.sql'); - } - - if ($aCMDResult['no-partitions']) { - pgsqlRunScript($aCMDResult, 'update country_name set partition = 0'); - } - - // the following will be needed by create_functions later but - // is only defined in the subsequently called T - // Create dummies here that will be overwritten by the proper - // versions in create-tables. - pgsqlRunScript($aCMDResult, 'CREATE TABLE IF NOT EXISTS place_boundingbox ()'); - pgsqlRunScript($aCMDResult, 'CREATE TYPE wikipedia_article_match AS ()', false); - } - return $aDSNInfo; -} - -function importData($aCMDResult, $iCacheMemory, $aDSNInfo) -{ - info('Import data'); - - $osm2pgsql = CONST_Osm2pgsql_Binary; - if (!file_exists($osm2pgsql)) { - echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; - echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '$osm2pgsql'"); - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; - } - if (CONST_Tablespace_Osm2pgsql_Data) - $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; - if (CONST_Tablespace_Osm2pgsql_Index) - $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; - if (CONST_Tablespace_Place_Data) - $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; - if (CONST_Tablespace_Place_Index) - $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; - $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.$iCacheMemory; - $osm2pgsql .= ' -P '.$aDSNInfo['port']; - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $osm2pgsql .= ' -U ' . $aDSNInfo['username']; - } - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $osm2pgsql .= ' -H ' . $aDSNInfo['hostspec']; - } - - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; - runWithEnv($osm2pgsql, $aProcEnv); - - $oDB =& getDB(); - if (!$aCMDResult['ignore-errors'] && !chksql($oDB->getRow('select * from place limit 1'))) { - fail('No Data'); - } -} - -function createFunctions($aCMDResult) -{ - info('Create Functions'); - - if (!checkModulePresence()) { - fail('error loading nominatim.so module'); - } - - createSqlFunctions($aCMDResult); -} - -function createTables($aCMDResult) -{ - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($aCMDResult, $sTemplate, false); - - // re-run the functions - info('Recreate Functions'); - createSqlFunctions($aCMDResult); -} - -function createPartitionTables($aCMDResult) -{ - info('Create Partition Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); - $sTemplate = replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - pgsqlRunPartitionScript($aCMDResult, $sTemplate); -} - -function createPartitionFunctions($aCMDResult) -{ - info('Create Partition Functions'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); - - pgsqlRunPartitionScript($aCMDResult, $sTemplate); -} - -function importWikipediaArticles() -{ - $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; - $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; - if (file_exists($sWikiArticlesFile)) { - info('Importing wikipedia articles'); - pgsqlRunDropAndRestore($sWikiArticlesFile); - } else { - warn('wikipedia article dump file not found - places will have default importance'); - } - if (file_exists($sWikiRedirectsFile)) { - info('Importing wikipedia redirects'); - pgsqlRunDropAndRestore($sWikiRedirectsFile); - } else { - warn('wikipedia redirect dump file not found - some place importance values may be missing'); - } -} - -function loadData($aCMDResult, $iInstances) -{ - info('Drop old Data'); - - $oDB =& getDB(); - if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection)); - echo '.'; - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($oDB->getCol($sSQL)); - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; - foreach ($aPartitions as $sPartition) { - if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection)); - echo '.'; - } - - // used by getorcreate_word_id to ignore frequent partial words - $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - echo ".\n"; - - // pre-create the word list - if (!$aCMDResult['disable-token-precalc']) { - info('Loading word list'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/words.sql'); - } - - info('Load Data'); - $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; - - $aDBInstances = array(); - $iLoadThreads = max(1, $iInstances - 1); - for ($i = 0; $i < $iLoadThreads; $i++) { - $aDBInstances[$i] =& getDB(true); - $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; - $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; - $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; - $sSQL .= ' and ST_IsValid(geometry)'; - if ($aCMDResult['verbose']) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$i]->connection)); - } - } - // last thread for interpolation lines - $aDBInstances[$iLoadThreads] =& getDB(true); - $sSQL = 'insert into location_property_osmline'; - $sSQL .= ' (osm_id, address, linegeo)'; - $sSQL .= ' SELECT osm_id, address, geometry from place where '; - $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($aCMDResult['verbose']) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); - } - - $bFailed = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { - $resultStatus = pg_result_status($hPGresult); - // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, - // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, - // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; - if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { - $resultError = pg_result_error($hPGresult); - echo '-- error text ' . $i . ': ' . $resultError . "\n"; - $bFailed = true; - } - } - } - if ($bFailed) { - fail('SQL errors loading placex and/or location_property_osmline tables'); - } - echo "\n"; - info('Reanalysing database'); - pgsqlRunScript($aCMDResult, 'ANALYSE'); - - $sDatabaseDate = getDatabaseDate($oDB); - pg_query($oDB->connection, 'TRUNCATE import_status'); - if ($sDatabaseDate === false) { - warn('could not determine database date.'); - } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - pg_query($oDB->connection, $sSQL); - echo "Latest data imported from $sDatabaseDate.\n"; - } -} - -function importTigerData($aCMDResult, $iInstances) -{ - info('Import Tiger data'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($aCMDResult, $sTemplate, false); - - $aDBInstances = array(); - for ($i = 0; $i < $iInstances; $i++) { - $aDBInstances[$i] =& getDB(true); - } - - foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { - echo $sFile.': '; - $hFile = fopen($sFile, 'r'); - $sSQL = fgets($hFile, 100000); - $iLines = 0; - - while (true) { - for ($i = 0; $i < $iInstances; $i++) { - if (!pg_connection_busy($aDBInstances[$i]->connection)) { - while (pg_get_result($aDBInstances[$i]->connection)); - $sSQL = fgets($hFile, 100000); - if (!$sSQL) break 2; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - $iLines++; - if ($iLines == 1000) { - echo '.'; - $iLines = 0; - } - } - } - usleep(10); - } - - fclose($hFile); - - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i < $iInstances; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; - } - usleep(10); - } - echo "\n"; - } - - info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($aCMDResult, $sTemplate, false); -} - -function calculatePostcodes($aCMDResult) -{ - info('Calculate Postcodes'); - $oDB =& getDB(); - if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) { - fail(pg_last_error($oDB->connection)); - } - - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; - $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; - $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; - $sSQL .= ' FROM placex'; - $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; - $sSQL .= ' AND geometry IS NOT null'; - $sSQL .= ' GROUP BY country_code, pc'; - - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - - if (CONST_Use_Extra_US_Postcodes) { - // only add postcodes that are not yet available in OSM - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; - $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; - $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'us')"; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - } - - // add missing postcodes for GB (if available) - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; - $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'gb')"; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - - if (!$aCMDResult['all']) { - $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; - $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - } - $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; - $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; - - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } -} - -function index($aCMDResult, $aDSNInfo, $iInstances) -{ - $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile; - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sBaseCmd .= ' -H ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sBaseCmd .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - info('Index ranks 0 - 4'); - $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$aCMDResult['index-noanalyse']) pgsqlRunScript($aCMDResult, 'ANALYSE'); - info('Index ranks 5 - 25'); - $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$aCMDResult['index-noanalyse']) pgsqlRunScript($aCMDResult, 'ANALYSE'); - info('Index ranks 26 - 30'); - $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - info('Index postcodes'); - $oDB =& getDB(); - $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); -} - -function createSearchIndices($aCMDResult) -{ - info('Create Search indices'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - pgsqlRunScript($aCMDResult, $sTemplate); -} - -function createCountryNames($aCMDResult) -{ - info('Create search index for default country names'); - - pgsqlRunScript($aCMDResult, "select getorcreate_country(make_standard_name('uk'), 'gb')"); - pgsqlRunScript($aCMDResult, "select getorcreate_country(make_standard_name('united states'), 'us')"); - pgsqlRunScript($aCMDResult, 'select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); - pgsqlRunScript($aCMDResult, "select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; - if (CONST_Languages) { - $sSQL .= 'in '; - $sDelim = '('; - foreach (explode(',', CONST_Languages) as $sLang) { - $sSQL .= $sDelim."'name:$sLang'"; - $sDelim = ','; - } - $sSQL .= ')'; - } else { - // all include all simple name tags - $sSQL .= "like 'name:%'"; - } - $sSQL .= ') v'; - pgsqlRunScript($aCMDResult, $sSQL); -} - -function drop($aCMDResult) -{ - info('Drop tables only required for updates'); - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. - - - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*' - ); - - $oDB =& getDB(); - $aDropTables = array(); - $aHaveTables = chksql($oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - - foreach ($aDropTables as $sDrop) { - if ($aCMDResult['verbose']) echo "dropping table $sDrop\n"; - @pg_query($oDB->connection, "DROP TABLE $sDrop CASCADE"); - // ignore warnings/errors as they might be caused by a table having - // been deleted already by CASCADE - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - if ($aCMDResult['verbose']) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; - unlink(CONST_Osm2pgsql_Flatnode_File); - } -} - -function didSomething($bDidSomething) -{ - if (!$bDidSomething) { - showUsage($aCMDOptions, true); - } else { - echo "Summary of warnings:\n\n"; - repeatWarnings(); - echo "\n"; - info('Setup finished.'); - } -} - -// ********************************* - -function pgsqlRunScriptFile($aCMDResult, $sFilename) -{ - if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; - if (!$aCMDResult['verbose']) { - $sCMD .= ' -q'; - } - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sCMD .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - $ahGzipPipes = null; - if (preg_match('/\\.gz$/', $sFilename)) { - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); - if (!is_resource($hGzipProcess)) fail('unable to start zcat'); - $aReadPipe = $ahGzipPipes[1]; - fclose($ahGzipPipes[0]); - } else { - $sCMD .= ' -f '.$sFilename; - $aReadPipe = array('pipe', 'r'); - } - $aDescriptors = array( - 0 => $aReadPipe, - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); - if (!is_resource($hProcess)) fail('unable to start pgsql'); - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - $iReturn = proc_close($hProcess); - if ($iReturn > 0) { - fail("pgsql returned with error code ($iReturn)"); - } - if ($ahGzipPipes) { - fclose($ahGzipPipes[1]); - proc_close($hGzipProcess); - } -} - -function pgsqlRunScript($aCMDResult, $sScript, $bfatal = true) -{ - runSQLScript( - $sScript, - $bfatal, - $aCMDResult['verbose'], - $aCMDResult['ignore-errors'] - ); -} - -function pgsqlRunPartitionScript($aCMDResult, $sTemplate) -{ - global $aCMDResult; - $oDB =& getDB(); - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($oDB->getCol($sSQL)); - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; - - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach ($aMatches as $aMatch) { - $sResult = ''; - foreach ($aPartitions as $sPartitionName) { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - - pgsqlRunScript($aCMDResult, $sTemplate); -} - -function pgsqlRunRestoreData($sDumpFile) -{ - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile; - - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); - if (!is_resource($hProcess)) fail('unable to start pg_restore'); - - fclose($ahPipes[0]); - - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - - $iReturn = proc_close($hProcess); -} - -function pgsqlRunDropAndRestore($sDumpFile) -{ - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile; - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sCMD .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $iReturn = runWithEnv($sCMD, $aProcEnv); -} - -function passthpassthruCheckReturn($sCmd) -{ - $iResult = -1; - passthru($sCmd, $iResult); -} - -function replaceTablespace($sTemplate, $sTablespace, $sSql) -{ - if ($sTablespace) { - $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); - } else { - $sSql = str_replace($sTemplate, '', $sSql); - } - - return $sSql; -} - -function createSqlFunctions($aCMDResult) -{ - $sModulePath = CONST_Database_Module_Path; - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', $sModulePath, $sTemplate); - if ($aCMDResult['enable-diff-updates']) { - $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); - } - if ($aCMDResult['enable-debug-statements']) { - $sTemplate = str_replace('--DEBUG:', '', $sTemplate); - } - if (CONST_Limit_Reindexing) { - $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); - } - if (!CONST_Use_US_Tiger_Data) { - $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); - } - if (!CONST_Use_Aux_Location_data) { - $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); - } - pgsqlRunScript($aCMDResult, $sTemplate); -} - -function checkModulePresence() -{ - // Try accessing the C module, so we know early if something is wrong - // and can simply error out. - $sModulePath = CONST_Database_Module_Path; - $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; - $sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; - $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; - - $oDB =& getDB(); - $oResult = $oDB->query($sSQL); - - $bResult = true; - - if (PEAR::isError($oResult)) { - echo "\nERROR: Failed to load nominatim module. Reason:\n"; - echo $oResult->userinfo."\n\n"; - $bResult = false; - } - - return $bResult; -} - -function createSetupArgvArray() -{ - $aCMDOptions - = array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - ); - return $aCMDOptions; -} +query($sSQL); + + $bResult = true; + + if (PEAR::isError($oResult)) { + echo "\nERROR: Failed to load nominatim module. Reason:\n"; + echo $oResult->userinfo."\n\n"; + $bResult = false; + } + + return $bResult; +} + +function createSetupArgvArray() { + $aCMDOptions + = array( + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), + ); + return $aCMDOptions; +} diff --git a/utils/setup.php b/utils/setup.php index 449dd4da..6afe196c 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -1,99 +1,135 @@ -#!@PHP_BIN@ -Cq -indirect via init-cmd.php -> /lib/cmd.php for runWithEnv, getCmdOpt +// ->indirect via init-cmd.php -> /lib/init.php -> db.php for &getDB() + +include_once(CONST_BasePath.'/lib/setup_functions.php'); +include_once(CONST_BasePath.'/lib/setup_functions.php'); +ini_set('memory_limit', '800M'); + + +$aCMDOptions = createSetupArgvArray(); + +// $aCMDOptions passed to getCmdOpt by reference +getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); + + +$bDidSomething = false; + +//******************************************************* +// Making some sanity check: +// Check if osm-file is set and points to a valid file +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + // to remain in /lib/setup_functions.php function + checkInFile($aCMDResult['osm-file']); + echo $aCMDResult['osm-file']; +} + +// osmosis init is no longer supported +if ($aCMDResult['osmosis-init']) { + $bDidSomething = true; + echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n"; +} + +// ****************************************************** +// instantiate Setup class +$cSetup = new SetupFunctions ($aCMDResult); +if ($aCMDResult['create-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createDB(); +} + +// ******************************************************* +// go through complete process if 'all' is selected or start selected functions +if ($aCMDResult['setup-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> setupDB(); +} + +// Try accessing the C module, so we know early if something is wrong +if (!checkModulePresence()) { + fail('error loading nominatim.so module'); +} + +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> importData($aCMDResult['osm-file']); +} + +if ($aCMDResult['create-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createFunctions(); +} + +if ($aCMDResult['create-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createTables(); +} + +if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createPartitionTables(); +} + +if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createPartitionFunctions(); +} +/* +if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> importWikipediaArticles(); +} +*/ +if ($aCMDResult['load-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> loadData($aCMDResult['disable-token-precalc']); +} + +if ($aCMDResult['import-tiger-data']) { + $bDidSomething = true; + $cSetup -> importTigerData(); +} + +if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> calculatePostcodes($aCMDResult['all']); +} + +if ($aCMDResult['index'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> index($aCMDResult['index-noanalyse']); +} + +if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createSearchIndices(); +} + +if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createCountryNames($aCMDResult); +} + +if ($aCMDResult['drop']) { + $bDidSomething = true; + $cSetup -> drop($aCMDResult); +} + +// ****************************************************** +// If we did something, repeat the warnings +if (!$bDidSomething) { + showUsage($aCMDOptions, true); +} else { + echo "Summary of warnings:\n\n"; + repeatWarnings(); + echo "\n"; + info('Setup finished.'); +} + + + diff --git a/utils/setupClass.php b/utils/setupClass.php new file mode 100755 index 00000000..03a0205b --- /dev/null +++ b/utils/setupClass.php @@ -0,0 +1,823 @@ +iInstances = isset($aCMDResult['threads']) + ? $aCMDResult['threads'] + : (min(16, getProcessorCount()) - 1); + + + if ($this->iInstances < 1) { + $this->iInstances = 1; + warn('resetting threads to '.$this->iInstances); + } + + // Assume we can steal all the cache memory in the box (unless told otherwise) + if (isset($aCMDResult['osm2pgsql-cache'])) { + $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; + } else { + $this->iCacheMemory = getCacheMemoryMB(); + } + + $this->sModulePath = CONST_Database_Module_Path; + info('module path: ' . $this->sModulePath); + + // prepares DB for import or update, sets the Data Source Name + $this->aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; + + // setting member variables based on command line options stored in $aCMDResult + $this->sVerbose = $aCMDResult['verbose']; + $this->sIgnoreErrors = $aCMDResult['ignore-errors']; + $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; + $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; + $this->bNoPartitions = $aCMDResult['no-partitions']; + + } + + public function createDB(){ + info('Create DB'); + $sDB = DB::connect(CONST_Database_DSN, false); + if (!PEAR::isError($sDB)) { + fail('database already exists ('.CONST_Database_DSN.')'); + } + + $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; + } + + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec']; + } + + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + $result = runWithEnv($sCreateDBCmd, $aProcEnv); + if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); + } + + public function setupDB() { + info('Setup DB'); + $this->oDB =& getDB(); + + $fPostgresVersion = getPostgresVersion($this->oDB); + echo 'Postgres version found: '.$fPostgresVersion."\n"; + + if ($fPostgresVersion < 9.1) { + fail('Minimum supported version of Postgresql is 9.1.'); + } + + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); + + // For extratags and namedetails the hstore_to_json converter is + // needed which is only available from Postgresql 9.3+. For older + // versions add a dummy function that returns nothing. + $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); + + if ($iNumFunc == 0) { + $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); + warn('Postgresql is too old. extratags and namedetails API not available.'); + } + + + $fPostgisVersion = getPostgisVersion($this->oDB); + echo 'Postgis version found: '.$fPostgisVersion."\n"; + + if ($fPostgisVersion < 2.1) { + // Functions were renamed in 2.1 and throw an annoying deprecation warning + $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); + $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); + } + if ($fPostgisVersion < 2.2) { + $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); + } + + $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); + if ($i == 0) { + echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; + echo "\n createuser ".CONST_Database_Web_User."\n\n"; + exit(1); + } + + if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { + echo 'Error: you need to download the country_osm_grid first:'; + echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; + exit(1); + } + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); + + + if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); + } else { + warn('external UK postcode table not found.'); + } + + if (CONST_Use_Extra_US_Postcodes) { + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); + } + + if ($this->bNoPartitions) { + $this->pgsqlRunScript('update country_name set partition = 0'); + } + + // the following will be needed by create_functions later but + // is only defined in the subsequently called T + // Create dummies here that will be overwritten by the proper + // versions in create-tables. + $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); + $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); + } + + public function importData($sOSMFile) { + info('Import data'); + + $osm2pgsql = CONST_Osm2pgsql_Binary; + if (!file_exists($osm2pgsql)) { + echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; + echo "Normally you should not need to set this manually.\n"; + fail("osm2pgsql not found in '$osm2pgsql'"); + } + + + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; + } + + if (CONST_Tablespace_Osm2pgsql_Data) + $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; + if (CONST_Tablespace_Osm2pgsql_Index) + $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; + if (CONST_Tablespace_Place_Data) + $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; + if (CONST_Tablespace_Place_Index) + $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; + $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; + $osm2pgsql .= ' -C '.$this->iCacheMemory; + $osm2pgsql .= ' -P '.$this->aDSNInfo['port']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile; + runWithEnv($osm2pgsql, $aProcEnv); + if ($this->oDB == null) $this->oDB =& getDB(); + if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { + fail('No Data'); + } + } + + public function createFunctions() { + info('Create Functions'); + + $this->createSqlFunctions(); + } + + public function createTables() { + info('Create Tables'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunScript($sTemplate, false); + + // re-run the functions + info('Recreate Functions'); + $this->createSqlFunctions(); + } + + public function createPartitionTables() { + info('Create Partition Tables'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function createPartitionFunctions() { + info('Create Partition Functions'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function importWikipediaArticles() { + $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; + $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; + if (file_exists($sWikiArticlesFile)) { + info('Importing wikipedia articles'); + $this->pgsqlRunDropAndRestore($sWikiArticlesFile); + } else { + warn('wikipedia article dump file not found - places will have default importance'); + } + if (file_exists($sWikiRedirectsFile)) { + info('Importing wikipedia redirects'); + $this->pgsqlRunDropAndRestore($sWikiRedirectsFile); + } else { + warn('wikipedia redirect dump file not found - some place importance values may be missing'); + } + echo ' finish wikipedia'; + } + + public function loadData($bDisableTokenPrecalc) { + info('Drop old Data'); + + if ($this->oDB == null) $this->oDB =& getDB(); + + if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) $aPartitions[] = 0; + foreach ($aPartitions as $sPartition) { + if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection)); + echo '.'; + } + + // used by getorcreate_word_id to ignore frequent partial words + $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; + $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + echo ".\n"; + + // pre-create the word list + if (!$bDisableTokenPrecalc) { + info('Loading word list'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); + } + + info('Load Data'); + $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; + $aDBInstances = array(); + $iLoadThreads = max(1, $this->iInstances - 1); + for ($i = 0; $i < $iLoadThreads; $i++) { + $aDBInstances[$i] =& getDB(true); + $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; + $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; + $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; + $sSQL .= ' and ST_IsValid(geometry)'; + if ($this->sVerbose) echo "$sSQL\n"; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$i]->connection)); + } + } + + // last thread for interpolation lines + + $aDBInstances[$iLoadThreads] =& getDB(true); + $sSQL = 'insert into location_property_osmline'; + $sSQL .= ' (osm_id, address, linegeo)'; + $sSQL .= ' SELECT osm_id, address, geometry from place where '; + $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; + if ($this->sVerbose) echo "$sSQL\n"; + if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); + } + + $bFailed = false; + for ($i = 0; $i <= $iLoadThreads; $i++) { + while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { + $resultStatus = pg_result_status($hPGresult); + // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, + // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, + // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR + echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; + if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { + $resultError = pg_result_error($hPGresult); + echo '-- error text ' . $i . ': ' . $resultError . "\n"; + $bFailed = true; + } + } + } + if ($bFailed) { + fail('SQL errors loading placex and/or location_property_osmline tables'); + } + echo "\n"; + info('Reanalysing database'); + $this->pgsqlRunScript('ANALYSE'); + + $sDatabaseDate = getDatabaseDate($this->oDB); + pg_query($this->oDB->connection, 'TRUNCATE import_status'); + if ($sDatabaseDate === false) { + warn('could not determine database date.'); + } else { + $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; + pg_query($this->oDB->connection, $sSQL); + echo "Latest data imported from $sDatabaseDate.\n"; + } + } + + public function importTigerData() { + info('Import Tiger data'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + + $aDBInstances = array(); + for ($i = 0; $i < $this->iInstances; $i++) { + $aDBInstances[$i] =& getDB(true); + } + + foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { + echo $sFile.': '; + $hFile = fopen($sFile, 'r'); + $sSQL = fgets($hFile, 100000); + $iLines = 0; + while (true) { + for ($i = 0; $i < $this->iInstances; $i++) { + if (!pg_connection_busy($aDBInstances[$i]->connection)) { + while (pg_get_result($aDBInstances[$i]->connection)); + $sSQL = fgets($hFile, 100000); + if (!$sSQL) break 2; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + $iLines++; + if ($iLines == 1000) { + echo '.'; + $iLines = 0; + } + } + } + usleep(10); + } + fclose($hFile); + + $bAnyBusy = true; + while ($bAnyBusy) { + $bAnyBusy = false; + for ($i = 0; $i < $this->iInstances; $i++) { + if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; + } + usleep(10); + } + echo "\n"; + } + + info('Creating indexes on Tiger data'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + } + + public function calculatePostcodes($bCMDResultAll) { + info('Calculate Postcodes'); + if ($this->oDB == null) $this->oDB =& getDB(); + if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { + fail(pg_last_error($this->oDB->connection)); + } + + + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; + $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; + $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; + $sSQL .= ' FROM placex'; + $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; + $sSQL .= ' AND geometry IS NOT null'; + $sSQL .= ' GROUP BY country_code, pc'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + if (CONST_Use_Extra_US_Postcodes) { + // only add postcodes that are not yet available in OSM + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; + $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; + $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'us')"; + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + } + + // add missing postcodes for GB (if available) + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; + $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'gb')"; + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + + if (!$bCMDResultAll) { + $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; + $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; + $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + + public function index($bIndexNoanalyse) { + $sOutputFile = ''; + $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sBaseCmd .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + info('Index ranks 0 - 4'); + $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); + info('Index ranks 5 - 25'); + $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); + info('Index ranks 26 - 30'); + $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + info('Index postcodes'); + if ($this->oDB == null) $this->oDB =& getDB(); + $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + } + + public function createSearchIndices() { + info('Create Search indices'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate); + } + + public function createCountryNames() { + info('Create search index for default country names'); + + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); + $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); + $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); + $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; + if (CONST_Languages) { + $sSQL .= 'in '; + $sDelim = '('; + foreach (explode(',', CONST_Languages) as $sLang) { + $sSQL .= $sDelim."'name:$sLang'"; + $sDelim = ','; + } + $sSQL .= ')'; + } else { + // all include all simple name tags + $sSQL .= "like 'name:%'"; + } + $sSQL .= ') v'; + $this->pgsqlRunScript($sSQL); + } + + public function drop() { + info('Drop tables only required for updates'); + + // The implementation is potentially a bit dangerous because it uses + // a positive selection of tables to keep, and deletes everything else. + // Including any tables that the unsuspecting user might have manually + // created. USE AT YOUR OWN PERIL. + // tables we want to keep. everything else goes. + $aKeepTables = array( + '*columns', + 'import_polygon_*', + 'import_status', + 'place_addressline', + 'location_postcode', + 'location_property*', + 'placex', + 'search_name', + 'seq_*', + 'word', + 'query_log', + 'new_query_log', + 'spatial_ref_sys', + 'country_name', + 'place_classtype_*' + ); + + if ($this->oDB = null) $this->oDB =& getDB(); + $aDropTables = array(); + $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); + + foreach ($aHaveTables as $sTable) { + $bFound = false; + foreach ($aKeepTables as $sKeep) { + if (fnmatch($sKeep, $sTable)) { + $bFound = true; + break; + } + } + if (!$bFound) array_push($aDropTables, $sTable); + } + foreach ($aDropTables as $sDrop) { + if ($this->sVerbose) echo "dropping table $sDrop\n"; + @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); + // ignore warnings/errors as they might be caused by a table having + // been deleted already by CASCADE + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; + unlink(CONST_Osm2pgsql_Flatnode_File); + } + } + + private function pgsqlRunDropAndRestore($sDumpFile) { + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; + $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" + } + + private function pgsqlRunScript($sScript, $bfatal = true) { + runSQLScript( + $sScript, + $bfatal, + $this->sVerbose, + $this->sIgnoreErrors + ); + } + + private function createSqlFunctions() { + $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); + $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); + if ($this->bEnableDiffUpdates) { + $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); + } + if ($this->bEnableDebugStatements) { + $sTemplate = str_replace('--DEBUG:', '', $sTemplate); + } + if (CONST_Limit_Reindexing) { + $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); + } + if (!CONST_Use_US_Tiger_Data) { + $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); + } + if (!CONST_Use_Aux_Location_data) { + $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); + } + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunPartitionScript($sTemplate) { + if ($this->oDB == null) $this->oDB =& getDB(); + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) $aPartitions[] = 0; + + preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); + foreach ($aMatches as $aMatch) { + $sResult = ''; + foreach ($aPartitions as $sPartitionName) { + $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); + } + $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); + } + + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunScriptFile($sFilename) { + if (!file_exists($sFilename)) fail('unable to find '.$sFilename); + + $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; + if (!$this->sVerbose) { + $sCMD .= ' -q'; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $ahGzipPipes = null; + if (preg_match('/\\.gz$/', $sFilename)) { + $aDescriptors = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); + if (!is_resource($hGzipProcess)) fail('unable to start zcat'); + $aReadPipe = $ahGzipPipes[1]; + fclose($ahGzipPipes[0]); + } else { + $sCMD .= ' -f '.$sFilename; + $aReadPipe = array('pipe', 'r'); + } + $aDescriptors = array( + 0 => $aReadPipe, + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $ahPipes = null; + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); + if (!is_resource($hProcess)) fail('unable to start pgsql'); + // TODO: error checking + while (!feof($ahPipes[1])) { + echo fread($ahPipes[1], 4096); + } + fclose($ahPipes[1]); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) { + fail("pgsql returned with error code ($iReturn)"); + } + if ($ahGzipPipes) { + fclose($ahGzipPipes[1]); + proc_close($hGzipProcess); + } + } + private function replaceTablespace($sTemplate, $sTablespace, $sSql) { + if ($sTablespace) { + $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); + } else { + $sSql = str_replace($sTemplate, '', $sSql); + } + return $sSql; + } +} -- 2.39.5