From: ThomasBarris Date: Fri, 31 Aug 2018 19:31:38 +0000 (+0200) Subject: small fixes on setup.php and a bring update.php to work X-Git-Tag: v3.3.0~84^2~8 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/a3b4f80c996056136828560e8e5fb26db367f238?hp=b2f3cfde0bb3d6a02e248272c3101970d7cfeb2e small fixes on setup.php and a bring update.php to work --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 5812883d..af6b9a92 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,6 @@ set(CUSTOMFILES utils/query.php utils/server_compare.php utils/setup.php - utils/setupClass.php utils/specialphrases.php utils/update.php utils/warm.php diff --git a/utils/setupClass.php b/lib/SetupClass.php old mode 100755 new mode 100644 similarity index 72% rename from utils/setupClass.php rename to lib/SetupClass.php index c5161542..d50d9905 --- a/utils/setupClass.php +++ b/lib/SetupClass.php @@ -1,844 +1,950 @@ -iInstances = isset($aCMDResult['threads']) - ? $aCMDResult['threads'] - : (min(16, getProcessorCount()) - 1); - - - if ($this->iInstances < 1) { - $this->iInstances = 1; - warn('resetting threads to '.$this->iInstances); - } - - // Assume we can steal all the cache memory in the box (unless told otherwise) - if (isset($aCMDResult['osm2pgsql-cache'])) { - $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; - } else { - $this->iCacheMemory = getCacheMemoryMB(); - } - - $this->sModulePath = CONST_Database_Module_Path; - info('module path: ' . $this->sModulePath); - - // prepares DB for import or update, sets the Data Source Name - $this->aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; - - // setting member variables based on command line options stored in $aCMDResult - $this->sVerbose = $aCMDResult['verbose']; - $this->sIgnoreErrors = $aCMDResult['ignore-errors']; - $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; - $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; - $this->bNoPartitions = $aCMDResult['no-partitions']; - } - - public function createDB() - { - info('Create DB'); - $sDB = DB::connect(CONST_Database_DSN, false); - if (!PEAR::isError($sDB)) { - fail('database already exists ('.CONST_Database_DSN.')'); - } - - $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database']; - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; - } - - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec']; - } - - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - - $result = runWithEnv($sCreateDBCmd, $aProcEnv); - if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); - } - - public function setupDB() - { - info('Setup DB'); - $this->oDB =& getDB(); - - $fPostgresVersion = getPostgresVersion($this->oDB); - echo 'Postgres version found: '.$fPostgresVersion."\n"; - - if ($fPostgresVersion < 9.1) { - fail('Minimum supported version of Postgresql is 9.1.'); - } - - $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); - $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); - - // For extratags and namedetails the hstore_to_json converter is - // needed which is only available from Postgresql 9.3+. For older - // versions add a dummy function that returns nothing. - $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); - - if ($iNumFunc == 0) { - $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); - warn('Postgresql is too old. extratags and namedetails API not available.'); - } - - - $fPostgisVersion = getPostgisVersion($this->oDB); - echo 'Postgis version found: '.$fPostgisVersion."\n"; - - if ($fPostgisVersion < 2.1) { - // Functions were renamed in 2.1 and throw an annoying deprecation warning - $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); - $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); - } - if ($fPostgisVersion < 2.2) { - $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); - } - - $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); - if ($i == 0) { - echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; - echo "\n createuser ".CONST_Database_Web_User."\n\n"; - exit(1); - } - - if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { - echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; - exit(1); - } - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); - - - if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); - } else { - warn('external UK postcode table not found.'); - } - - if (CONST_Use_Extra_US_Postcodes) { - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); - } - - if ($this->bNoPartitions) { - $this->pgsqlRunScript('update country_name set partition = 0'); - } - - // the following will be needed by create_functions later but - // is only defined in the subsequently called T - // Create dummies here that will be overwritten by the proper - // versions in create-tables. - $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); - $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); - } - - public function importData($sOSMFile) - { - info('Import data'); - - $osm2pgsql = CONST_Osm2pgsql_Binary; - if (!file_exists($osm2pgsql)) { - echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; - echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '$osm2pgsql'"); - } - - - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; - } - - if (CONST_Tablespace_Osm2pgsql_Data) - $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; - if (CONST_Tablespace_Osm2pgsql_Index) - $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; - if (CONST_Tablespace_Place_Data) - $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; - if (CONST_Tablespace_Place_Index) - $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; - $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.$this->iCacheMemory; - $osm2pgsql .= ' -P '.$this->aDSNInfo['port']; - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; - } - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - $osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile; - runWithEnv($osm2pgsql, $aProcEnv); - if ($this->oDB == null) $this->oDB =& getDB(); - if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { - fail('No Data'); - } - } - - public function createFunctions() - { - info('Create Functions'); - - $this->createSqlFunctions(); - } - - public function createTables() - { - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - $this->pgsqlRunScript($sTemplate, false); - } - - public function recreateFunction() - { - // re-run the functions - info('Recreate Functions'); - $this->createSqlFunctions(); - } - - public function createPartitionTables() - { - info('Create Partition Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); - $sTemplate = $this->replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - $this->pgsqlRunPartitionScript($sTemplate); - } - - public function createPartitionFunctions() - { - info('Create Partition Functions'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); - $this->pgsqlRunPartitionScript($sTemplate); - } - - public function importWikipediaArticles() - { - $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; - $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; - if (file_exists($sWikiArticlesFile)) { - info('Importing wikipedia articles'); - $this->pgsqlRunDropAndRestore($sWikiArticlesFile); - } else { - warn('wikipedia article dump file not found - places will have default importance'); - } - if (file_exists($sWikiRedirectsFile)) { - info('Importing wikipedia redirects'); - $this->pgsqlRunDropAndRestore($sWikiRedirectsFile); - } else { - warn('wikipedia redirect dump file not found - some place importance values may be missing'); - } - echo ' finish wikipedia'; - } - - public function loadData($bDisableTokenPrecalc) - { - info('Drop old Data'); - - if ($this->oDB == null) $this->oDB =& getDB(); - - if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($this->oDB->getCol($sSQL)); - if (!$this->bNoPartitions) $aPartitions[] = 0; - foreach ($aPartitions as $sPartition) { - if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection)); - echo '.'; - } - - // used by getorcreate_word_id to ignore frequent partial words - $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - echo ".\n"; - - // pre-create the word list - if (!$bDisableTokenPrecalc) { - info('Loading word list'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); - } - - info('Load Data'); - $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; - $aDBInstances = array(); - $iLoadThreads = max(1, $this->iInstances - 1); - for ($i = 0; $i < $iLoadThreads; $i++) { - $aDBInstances[$i] =& getDB(true); - $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; - $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; - $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; - $sSQL .= ' and ST_IsValid(geometry)'; - if ($this->sVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$i]->connection)); - } - } - - // last thread for interpolation lines - $aDBInstances[$iLoadThreads] =& getDB(true); - $sSQL = 'insert into location_property_osmline'; - $sSQL .= ' (osm_id, address, linegeo)'; - $sSQL .= ' SELECT osm_id, address, geometry from place where '; - $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($this->sVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); - } - - $bFailed = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { - $resultStatus = pg_result_status($hPGresult); - // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, - // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, - // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; - if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { - $resultError = pg_result_error($hPGresult); - echo '-- error text ' . $i . ': ' . $resultError . "\n"; - $bFailed = true; - } - } - } - if ($bFailed) { - fail('SQL errors loading placex and/or location_property_osmline tables'); - } - echo "\n"; - info('Reanalysing database'); - $this->pgsqlRunScript('ANALYSE'); - - $sDatabaseDate = getDatabaseDate($this->oDB); - pg_query($this->oDB->connection, 'TRUNCATE import_status'); - if ($sDatabaseDate === false) { - warn('could not determine database date.'); - } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - pg_query($this->oDB->connection, $sSQL); - echo "Latest data imported from $sDatabaseDate.\n"; - } - } - - public function importTigerData() - { - info('Import Tiger data'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - $this->pgsqlRunScript($sTemplate, false); - - $aDBInstances = array(); - for ($i = 0; $i < $this->iInstances; $i++) { - $aDBInstances[$i] =& getDB(true); - } - - foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { - echo $sFile.': '; - $hFile = fopen($sFile, 'r'); - $sSQL = fgets($hFile, 100000); - $iLines = 0; - while (true) { - for ($i = 0; $i < $this->iInstances; $i++) { - if (!pg_connection_busy($aDBInstances[$i]->connection)) { - while (pg_get_result($aDBInstances[$i]->connection)); - $sSQL = fgets($hFile, 100000); - if (!$sSQL) break 2; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - $iLines++; - if ($iLines == 1000) { - echo '.'; - $iLines = 0; - } - } - } - usleep(10); - } - fclose($hFile); - - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i < $this->iInstances; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; - } - usleep(10); - } - echo "\n"; - } - - info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - $this->pgsqlRunScript($sTemplate, false); - } - - public function calculatePostcodes($bCMDResultAll) - { - info('Calculate Postcodes'); - if ($this->oDB == null) $this->oDB =& getDB(); - if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { - fail(pg_last_error($this->oDB->connection)); - } - - - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; - $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; - $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; - $sSQL .= ' FROM placex'; - $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; - $sSQL .= ' AND geometry IS NOT null'; - $sSQL .= ' GROUP BY country_code, pc'; - - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - - if (CONST_Use_Extra_US_Postcodes) { - // only add postcodes that are not yet available in OSM - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; - $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; - $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'us')"; - if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - } - - // add missing postcodes for GB (if available) - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; - $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'gb')"; - if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - - if (!$bCMDResultAll) { - $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; - $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - } - $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; - $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; - - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - } - - public function index($bIndexNoanalyse) - { - $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P ' - .$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; - } - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sBaseCmd .= ' -U ' . $this->aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - - info('Index ranks 0 - 4'); - $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - info('Index ranks 5 - 25'); - $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - info('Index ranks 26 - 30'); - $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - info('Index postcodes'); - if ($this->oDB == null) $this->oDB =& getDB(); - $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - } - - public function createSearchIndices() - { - info('Create Search indices'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - $this->pgsqlRunScript($sTemplate); - } - - public function createCountryNames() - { - info('Create search index for default country names'); - - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); - $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); - $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' - .'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; - if (CONST_Languages) { - $sSQL .= 'in '; - $sDelim = '('; - foreach (explode(',', CONST_Languages) as $sLang) { - $sSQL .= $sDelim."'name:$sLang'"; - $sDelim = ','; - } - $sSQL .= ')'; - } else { - // all include all simple name tags - $sSQL .= "like 'name:%'"; - } - $sSQL .= ') v'; - $this->pgsqlRunScript($sSQL); - } - - public function drop() - { - info('Drop tables only required for updates'); - - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*' - ); - - if ($this->oDB = null) $this->oDB =& getDB(); - $aDropTables = array(); - $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - foreach ($aDropTables as $sDrop) { - if ($this->sVerbose) echo "dropping table $sDrop\n"; - @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); - // ignore warnings/errors as they might be caused by a table having - // been deleted already by CASCADE - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; - unlink(CONST_Osm2pgsql_Flatnode_File); - } - } - - private function pgsqlRunDropAndRestore($sDumpFile) - { - if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; - } - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCMD .= ' -U ' . $this->aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" - } - - private function pgsqlRunScript($sScript, $bfatal = true) - { - runSQLScript( - $sScript, - $bfatal, - $this->sVerbose, - $this->sIgnoreErrors - ); - } - - private function createSqlFunctions() - { - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); - if ($this->bEnableDiffUpdates) { - $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); - } - if ($this->bEnableDebugStatements) { - $sTemplate = str_replace('--DEBUG:', '', $sTemplate); - } - if (CONST_Limit_Reindexing) { - $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); - } - if (!CONST_Use_US_Tiger_Data) { - $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); - } - if (!CONST_Use_Aux_Location_data) { - $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); - } - $this->pgsqlRunScript($sTemplate); - } - - private function pgsqlRunPartitionScript($sTemplate) - { - if ($this->oDB == null) $this->oDB =& getDB(); - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($this->oDB->getCol($sSQL)); - if (!$this->bNoPartitions) $aPartitions[] = 0; - - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach ($aMatches as $aMatch) { - $sResult = ''; - foreach ($aPartitions as $sPartitionName) { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - - $this->pgsqlRunScript($sTemplate); - } - - private function pgsqlRunScriptFile($sFilename) - { - if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - - $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; - if (!$this->sVerbose) { - $sCMD .= ' -q'; - } - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; - } - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCMD .= ' -U ' . $this->aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - $ahGzipPipes = null; - if (preg_match('/\\.gz$/', $sFilename)) { - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); - if (!is_resource($hGzipProcess)) fail('unable to start zcat'); - $aReadPipe = $ahGzipPipes[1]; - fclose($ahGzipPipes[0]); - } else { - $sCMD .= ' -f '.$sFilename; - $aReadPipe = array('pipe', 'r'); - } - $aDescriptors = array( - 0 => $aReadPipe, - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); - if (!is_resource($hProcess)) fail('unable to start pgsql'); - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - $iReturn = proc_close($hProcess); - if ($iReturn > 0) { - fail("pgsql returned with error code ($iReturn)"); - } - if ($ahGzipPipes) { - fclose($ahGzipPipes[1]); - proc_close($hGzipProcess); - } - } - - private function replaceTablespace($sTemplate, $sTablespace, $sSql) - { - if ($sTablespace) { - $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); - } else { - $sSql = str_replace($sTemplate, '', $sSql); - } - return $sSql; - } -} +iInstances = isset($aCMDResult['threads']) + ? $aCMDResult['threads'] + : (min(16, getProcessorCount()) - 1); + + if ($this->iInstances < 1) { + $this->iInstances = 1; + warn('resetting threads to ' . $this->iInstances); + } + + // Assume we can steal all the cache memory in the box (unless told otherwise) + if (isset($aCMDResult['osm2pgsql-cache'])) { + $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; + } else { + $this->iCacheMemory = getCacheMemoryMB(); + } + + $this->sModulePath = CONST_Database_Module_Path; + info('module path: ' . $this->sModulePath); + + // prepares DB for import or update, sets the Data Source Name + $this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN); + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { + $this->aDSNInfo['port'] = 5432; + } + + // setting member variables based on command line options stored in $aCMDResult + $this->sVerbose = $aCMDResult['verbose']; + $this->sIgnoreErrors = $aCMDResult['ignore-errors']; + $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; + $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; + $this->bNoPartitions = $aCMDResult['no-partitions']; + } + + public function createDB() + { + info('Create DB'); + $sDB = \DB::connect(CONST_Database_DSN, false); + if (!\PEAR::isError($sDB)) { + fail('database already exists (' . CONST_Database_DSN . ')'); + } + + $sCreateDBCmd = 'createdb -E UTF-8 -p ' . $this->aDSNInfo['port'] . ' ' . $this->aDSNInfo['database']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; + } + + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec']; + } + + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + $result = runWithEnv($sCreateDBCmd, $aProcEnv); + if ($result != 0) { + fail('Error executing external command: ' . $sCreateDBCmd); + } + + } + + public function setupDB() + { + info('Setup DB'); + $this->oDB = &getDB(); + + $fPostgresVersion = getPostgresVersion($this->oDB); + echo 'Postgres version found: ' . $fPostgresVersion . "\n"; + + if ($fPostgresVersion < 9.1) { + fail('Minimum supported version of Postgresql is 9.1.'); + } + + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); + + // For extratags and namedetails the hstore_to_json converter is + // needed which is only available from Postgresql 9.3+. For older + // versions add a dummy function that returns nothing. + $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); + + if ($iNumFunc == 0) { + $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); + warn('Postgresql is too old. extratags and namedetails API not available.'); + } + + $fPostgisVersion = getPostgisVersion($this->oDB); + echo 'Postgis version found: ' . $fPostgisVersion . "\n"; + + if ($fPostgisVersion < 2.1) { + // Functions were renamed in 2.1 and throw an annoying deprecation warning + $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); + $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); + } + if ($fPostgisVersion < 2.2) { + $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); + } + + $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '" . CONST_Database_Web_User . "'")); + if ($i == 0) { + echo "\nERROR: Web user '" . CONST_Database_Web_User . "' does not exist. Create it with:\n"; + echo "\n createuser " . CONST_Database_Web_User . "\n\n"; + exit(1); + } + + if (!file_exists(CONST_ExtraDataPath . '/country_osm_grid.sql.gz')) { + echo 'Error: you need to download the country_osm_grid first:'; + echo "\n wget -O " . CONST_ExtraDataPath . "/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; + exit(1); + } + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_name.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_naturalearthdata.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_osm_grid.sql.gz'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_table.sql'); + + if (file_exists(CONST_BasePath . '/data/gb_postcode_data.sql.gz')) { + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_data.sql.gz'); + } else { + warn('external UK postcode table not found.'); + } + + if (CONST_Use_Extra_US_Postcodes) { + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/us_postcode.sql'); + } + + if ($this->bNoPartitions) { + $this->pgsqlRunScript('update country_name set partition = 0'); + } + + // the following will be needed by create_functions later but + // is only defined in the subsequently called T + // Create dummies here that will be overwritten by the proper + // versions in create-tables. + $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); + $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); + } + + public function importData($sOSMFile) + { + info('Import data'); + + $osm2pgsql = CONST_Osm2pgsql_Binary; + if (!file_exists($osm2pgsql)) { + echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; + echo "Normally you should not need to set this manually.\n"; + fail("osm2pgsql not found in '$osm2pgsql'"); + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + $osm2pgsql .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; + } + + if (CONST_Tablespace_Osm2pgsql_Data) { + $osm2pgsql .= ' --tablespace-slim-data ' . CONST_Tablespace_Osm2pgsql_Data; + } + + if (CONST_Tablespace_Osm2pgsql_Index) { + $osm2pgsql .= ' --tablespace-slim-index ' . CONST_Tablespace_Osm2pgsql_Index; + } + + if (CONST_Tablespace_Place_Data) { + $osm2pgsql .= ' --tablespace-main-data ' . CONST_Tablespace_Place_Data; + } + + if (CONST_Tablespace_Place_Index) { + $osm2pgsql .= ' --tablespace-main-index ' . CONST_Tablespace_Place_Index; + } + + $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; + $osm2pgsql .= ' -C ' . $this->iCacheMemory; + $osm2pgsql .= ' -P ' . $this->aDSNInfo['port']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $osm2pgsql .= ' -d ' . $this->aDSNInfo['database'] . ' ' . $sOSMFile; + runWithEnv($osm2pgsql, $aProcEnv); + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { + fail('No Data'); + } + } + + public function createFunctions() + { + info('Create Functions'); + + $this->createSqlFunctions(); + } + + public function createTables() + { + info('Create Tables'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/tables.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunScript($sTemplate, false); + } + + public function createPartitionTables() + { + info('Create Partition Tables'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-tables.src.sql'); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function createPartitionFunctions() + { + info('Create Partition Functions'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-functions.src.sql'); + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function importWikipediaArticles() + { + $sWikiArticlesFile = CONST_Wikipedia_Data_Path . '/wikipedia_article.sql.bin'; + $sWikiRedirectsFile = CONST_Wikipedia_Data_Path . '/wikipedia_redirect.sql.bin'; + if (file_exists($sWikiArticlesFile)) { + info('Importing wikipedia articles'); + $this->pgsqlRunDropAndRestore($sWikiArticlesFile); + } else { + warn('wikipedia article dump file not found - places will have default importance'); + } + if (file_exists($sWikiRedirectsFile)) { + info('Importing wikipedia redirects'); + $this->pgsqlRunDropAndRestore($sWikiRedirectsFile); + } else { + warn('wikipedia redirect dump file not found - some place importance values may be missing'); + } + echo ' finish wikipedia'; + } + + public function loadData($bDisableTokenPrecalc) + { + info('Drop old Data'); + + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + if (!pg_query($this->oDB->connection, 'TRUNCATE word')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) { + $aPartitions[] = 0; + } + + foreach ($aPartitions as $sPartition) { + if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_' . $sPartition)) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + } + + // used by getorcreate_word_id to ignore frequent partial words + $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; + $sSQL .= '$$ SELECT ' . CONST_Max_Word_Frequency . ' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + echo ".\n"; + + // pre-create the word list + if (!$bDisableTokenPrecalc) { + info('Loading word list'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/words.sql'); + } + + info('Load Data'); + $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; + $aDBInstances = array(); + $iLoadThreads = max(1, $this->iInstances - 1); + for ($i = 0; $i < $iLoadThreads; $i++) { + $aDBInstances[$i] = &getDB(true); + $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; + $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; + $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; + $sSQL .= ' and ST_IsValid(geometry)'; + if ($this->sVerbose) { + echo "$sSQL\n"; + } + + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$i]->connection)); + } + } + + // last thread for interpolation lines + $aDBInstances[$iLoadThreads] = &getDB(true); + $sSQL = 'insert into location_property_osmline'; + $sSQL .= ' (osm_id, address, linegeo)'; + $sSQL .= ' SELECT osm_id, address, geometry from place where '; + $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; + if ($this->sVerbose) { + echo "$sSQL\n"; + } + + if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); + } + + $bFailed = false; + for ($i = 0; $i <= $iLoadThreads; $i++) { + while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { + $resultStatus = pg_result_status($hPGresult); + // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, + // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, + // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR + echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; + if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { + $resultError = pg_result_error($hPGresult); + echo '-- error text ' . $i . ': ' . $resultError . "\n"; + $bFailed = true; + } + } + } + if ($bFailed) { + fail('SQL errors loading placex and/or location_property_osmline tables'); + } + echo "\n"; + info('Reanalysing database'); + $this->pgsqlRunScript('ANALYSE'); + + $sDatabaseDate = getDatabaseDate($this->oDB); + pg_query($this->oDB->connection, 'TRUNCATE import_status'); + if ($sDatabaseDate === false) { + warn('could not determine database date.'); + } else { + $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('" . $sDatabaseDate . "')"; + pg_query($this->oDB->connection, $sSQL); + echo "Latest data imported from $sDatabaseDate.\n"; + } + } + + public function importTigerData() + { + info('Import Tiger data'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_start.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + + $aDBInstances = array(); + for ($i = 0; $i < $this->iInstances; $i++) { + $aDBInstances[$i] = &getDB(true); + } + + foreach (glob(CONST_Tiger_Data_Path . '/*.sql') as $sFile) { + echo $sFile . ': '; + $hFile = fopen($sFile, 'r'); + $sSQL = fgets($hFile, 100000); + $iLines = 0; + while (true) { + for ($i = 0; $i < $this->iInstances; $i++) { + if (!pg_connection_busy($aDBInstances[$i]->connection)) { + while (pg_get_result($aDBInstances[$i]->connection)); + $sSQL = fgets($hFile, 100000); + if (!$sSQL) { + break 2; + } + + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + $iLines++; + if ($iLines == 1000) { + echo '.'; + $iLines = 0; + } + } + } + usleep(10); + } + fclose($hFile); + + $bAnyBusy = true; + while ($bAnyBusy) { + $bAnyBusy = false; + for ($i = 0; $i < $this->iInstances; $i++) { + if (pg_connection_busy($aDBInstances[$i]->connection)) { + $bAnyBusy = true; + } + + } + usleep(10); + } + echo "\n"; + } + + info('Creating indexes on Tiger data'); + $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_finish.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + } + + public function calculatePostcodes($bCMDResultAll) + { + info('Calculate Postcodes'); + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { + fail(pg_last_error($this->oDB->connection)); + } + + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; + $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; + $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; + $sSQL .= ' FROM placex'; + $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; + $sSQL .= ' AND geometry IS NOT null'; + $sSQL .= ' GROUP BY country_code, pc'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + if (CONST_Use_Extra_US_Postcodes) { + // only add postcodes that are not yet available in OSM + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; + $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; + $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'us')"; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + } + + // add missing postcodes for GB (if available) + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; + $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'gb')"; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + if (!$bCMDResultAll) { + $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; + $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; + $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + + public function index($bIndexNoanalyse) + { + $sOutputFile = ''; + $sBaseCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $this->aDSNInfo['database'] . ' -P ' + . $this->aDSNInfo['port'] . ' -t ' . $this->iInstances . $sOutputFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sBaseCmd .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + info('Index ranks 0 - 4'); + $iStatus = runWithEnv($sBaseCmd . ' -R 4', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) { + $this->pgsqlRunScript('ANALYSE'); + } + + info('Index ranks 5 - 25'); + $iStatus = runWithEnv($sBaseCmd . ' -r 5 -R 25', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) { + $this->pgsqlRunScript('ANALYSE'); + } + + info('Index ranks 26 - 30'); + $iStatus = runWithEnv($sBaseCmd . ' -r 26', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + info('Index postcodes'); + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + } + + public function createSearchIndices() + { + info('Create Search indices'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/indices.src.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate); + } + + public function createCountryNames() + { + info('Create search index for default country names'); + + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); + $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); + $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); + $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' + . 'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; + if (CONST_Languages) { + $sSQL .= 'in '; + $sDelim = '('; + foreach (explode(',', CONST_Languages) as $sLang) { + $sSQL .= $sDelim . "'name:$sLang'"; + $sDelim = ','; + } + $sSQL .= ')'; + } else { + // all include all simple name tags + $sSQL .= "like 'name:%'"; + } + $sSQL .= ') v'; + $this->pgsqlRunScript($sSQL); + } + + public function drop() + { + info('Drop tables only required for updates'); + + // The implementation is potentially a bit dangerous because it uses + // a positive selection of tables to keep, and deletes everything else. + // Including any tables that the unsuspecting user might have manually + // created. USE AT YOUR OWN PERIL. + // tables we want to keep. everything else goes. + $aKeepTables = array( + '*columns', + 'import_polygon_*', + 'import_status', + 'place_addressline', + 'location_postcode', + 'location_property*', + 'placex', + 'search_name', + 'seq_*', + 'word', + 'query_log', + 'new_query_log', + 'spatial_ref_sys', + 'country_name', + 'place_classtype_*', + ); + + if ($this->oDB = null) { + $this->oDB = &getDB(); + } + + $aDropTables = array(); + $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); + + foreach ($aHaveTables as $sTable) { + $bFound = false; + foreach ($aKeepTables as $sKeep) { + if (fnmatch($sKeep, $sTable)) { + $bFound = true; + break; + } + } + if (!$bFound) { + array_push($aDropTables, $sTable); + } + + } + foreach ($aDropTables as $sDrop) { + if ($this->sVerbose) { + echo "dropping table $sDrop\n"; + } + + @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); + // ignore warnings/errors as they might be caused by a table having + // been deleted already by CASCADE + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + if ($sVerbose) { + echo 'deleting ' . CONST_Osm2pgsql_Flatnode_File . "\n"; + } + + unlink(CONST_Osm2pgsql_Flatnode_File); + } + } + + private function pgsqlRunDropAndRestore($sDumpFile) + { + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { + $this->aDSNInfo['port'] = 5432; + } + + $sCMD = 'pg_restore -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'] . ' -Fc --clean ' . $sDumpFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" + } + + private function pgsqlRunScript($sScript, $bfatal = true) + { + runSQLScript( + $sScript, + $bfatal, + $this->sVerbose, + $this->sIgnoreErrors + ); + } + + private function createSqlFunctions() + { + $sTemplate = file_get_contents(CONST_BasePath . '/sql/functions.sql'); + $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); + if ($this->bEnableDiffUpdates) { + $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); + } + if ($this->bEnableDebugStatements) { + $sTemplate = str_replace('--DEBUG:', '', $sTemplate); + } + if (CONST_Limit_Reindexing) { + $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); + } + if (!CONST_Use_US_Tiger_Data) { + $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); + } + if (!CONST_Use_Aux_Location_data) { + $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); + } + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunPartitionScript($sTemplate) + { + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) { + $aPartitions[] = 0; + } + + preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); + foreach ($aMatches as $aMatch) { + $sResult = ''; + foreach ($aPartitions as $sPartitionName) { + $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); + } + $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); + } + + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunScriptFile($sFilename) + { + if (!file_exists($sFilename)) { + fail('unable to find ' . $sFilename); + } + + $sCMD = 'psql -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database']; + if (!$this->sVerbose) { + $sCMD .= ' -q'; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $ahGzipPipes = null; + if (preg_match('/\\.gz$/', $sFilename)) { + $aDescriptors = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a'), + ); + $hGzipProcess = proc_open('zcat ' . $sFilename, $aDescriptors, $ahGzipPipes); + if (!is_resource($hGzipProcess)) { + fail('unable to start zcat'); + } + + $aReadPipe = $ahGzipPipes[1]; + fclose($ahGzipPipes[0]); + } else { + $sCMD .= ' -f ' . $sFilename; + $aReadPipe = array('pipe', 'r'); + } + $aDescriptors = array( + 0 => $aReadPipe, + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a'), + ); + $ahPipes = null; + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); + if (!is_resource($hProcess)) { + fail('unable to start pgsql'); + } + + // TODO: error checking + while (!feof($ahPipes[1])) { + echo fread($ahPipes[1], 4096); + } + fclose($ahPipes[1]); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) { + fail("pgsql returned with error code ($iReturn)"); + } + if ($ahGzipPipes) { + fclose($ahGzipPipes[1]); + proc_close($hGzipProcess); + } + } + + private function replaceTablespace($sTemplate, $sTablespace, $sSql) + { + if ($sTablespace) { + $sSql = str_replace($sTemplate, 'TABLESPACE "' . $sTablespace . '"', $sSql); + } else { + $sSql = str_replace($sTemplate, '', $sSql); + } + return $sSql; + } +} diff --git a/lib/setup_functions.php b/lib/setup_functions.php index fec28f17..7b705168 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -1,80 +1,126 @@ -query($sSQL); - - $bResult = true; - - if (PEAR::isError($oResult)) { - echo "\nERROR: Failed to load nominatim module. Reason:\n"; - echo $oResult->userinfo."\n\n"; - $bResult = false; - } - - return $bResult; -} - -function createSetupArgvArray() -{ - $aCMDOptions - = array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - ); - return $aCMDOptions; -} +query($sSQL); + + $bResult = true; + + if (PEAR::isError($oResult)) { + echo "\nERROR: Failed to load nominatim module. Reason:\n"; + echo $oResult->userinfo . "\n\n"; + $bResult = false; + } + + return $bResult; +} + +// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) +// create and array +function createSetupArgvArray() +{ + $aCMDOptions + = array( + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), + ); + return $aCMDOptions; +} + +function createUpdateArgvArray() +{ + $aCMDOptions + = array( + 'Import / update / index osm data', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), + array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), + array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), + array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), + array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), + array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), + + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), + + array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), + array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + + array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), + array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), + array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), + array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), + + array('index', '', 0, 1, 0, 0, 'bool', 'Index'), + array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), + array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), + + array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), + array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), + array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), + + array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + ); + return $aCMDOptions; +} diff --git a/utils/setup.php b/utils/setup.php index 8eb90cfa..989d28e1 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -1,133 +1,133 @@ -#!@PHP_BIN@ -Cq -indirect via init-cmd.php -> /lib/cmd.php for runWithEnv, getCmdOpt -// ->indirect via init-cmd.php -> /lib/init.php -> db.php for &getDB() - -include_once(CONST_BasePath.'/lib/setup_functions.php'); -include_once(CONST_BasePath.'/lib/setup_functions.php'); -ini_set('memory_limit', '800M'); - - -$aCMDOptions = createSetupArgvArray(); - -// $aCMDOptions passed to getCmdOpt by reference -getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); - - -$bDidSomething = false; - -//******************************************************* -// Making some sanity check: -// Check if osm-file is set and points to a valid file -if ($aCMDResult['import-data'] || $aCMDResult['all']) { - // to remain in /lib/setup_functions.php function - checkInFile($aCMDResult['osm-file']); - echo $aCMDResult['osm-file']; -} - -// osmosis init is no longer supported -if ($aCMDResult['osmosis-init']) { - $bDidSomething = true; - echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n"; -} - -// ****************************************************** -// instantiate Setup class -$cSetup = new SetupFunctions($aCMDResult); -if ($aCMDResult['create-db'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createDB(); -} - -// ******************************************************* -// go through complete process if 'all' is selected or start selected functions -if ($aCMDResult['setup-db'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> setupDB(); -} - -// Try accessing the C module, so we know early if something is wrong -if (!checkModulePresence()) { - fail('error loading nominatim.so module'); -} - -if ($aCMDResult['import-data'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> importData($aCMDResult['osm-file']); -} - -if ($aCMDResult['create-functions'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createFunctions(); -} - -if ($aCMDResult['create-tables'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createTables(); - $cSetup -> recreateFunction(); -} - -if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createPartitionTables(); -} - -if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createPartitionFunctions(); -} - -if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> importWikipediaArticles(); -} - -if ($aCMDResult['load-data'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> loadData($aCMDResult['disable-token-precalc']); -} - -if ($aCMDResult['import-tiger-data']) { - $bDidSomething = true; - $cSetup -> importTigerData(); -} - -if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> calculatePostcodes($aCMDResult['all']); -} - -if ($aCMDResult['index'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> index($aCMDResult['index-noanalyse']); -} - -if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createSearchIndices(); -} - -if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createCountryNames($aCMDResult); -} - -if ($aCMDResult['drop']) { - $bDidSomething = true; - $cSetup -> drop($aCMDResult); -} - -// ****************************************************** -// If we did something, repeat the warnings -if (!$bDidSomething) { - showUsage($aCMDOptions, true); -} else { - echo "Summary of warnings:\n\n"; - repeatWarnings(); - echo "\n"; - info('Setup finished.'); -} +#!@PHP_BIN@ -Cq +indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt +// ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB() + +require_once CONST_BasePath . '/lib/setup_functions.php'; +ini_set('memory_limit', '800M'); + +use Nominatim\Setup\SetupFunctions as SetupFunctions; + +$aCMDOptions = createSetupArgvArray(); + +// $aCMDOptions passed to getCmdOpt by reference +getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); + +$bDidSomething = false; + +//******************************************************* +// Making some sanity check: +// Check if osm-file is set and points to a valid file +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + // to remain in /lib/setup_functions.php function + checkInFile($aCMDResult['osm-file']); + echo $aCMDResult['osm-file']; +} + +// osmosis init is no longer supported +if ($aCMDResult['osmosis-init']) { + $bDidSomething = true; + echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n"; +} + +// ****************************************************** +// instantiate Setup class +$cSetup = new SetupFunctions($aCMDResult); + +// ******************************************************* +// go through complete process if 'all' is selected or start selected functions +if ($aCMDResult['create-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createDB(); +} + +if ($aCMDResult['setup-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->setupDB(); +} + +// Try accessing the C module, so we know early if something is wrong +if (!checkModulePresence()) { + fail('error loading nominatim.so module'); +} + +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->importData($aCMDResult['osm-file']); +} + +if ($aCMDResult['create-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createFunctions(); +} + +if ($aCMDResult['create-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createTables(); + $cSetup->createFunctions(); +} + +if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createPartitionTables(); +} + +if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createPartitionFunctions(); +} + +if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->importWikipediaArticles(); +} + +if ($aCMDResult['load-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->loadData($aCMDResult['disable-token-precalc']); +} + +if ($aCMDResult['import-tiger-data']) { + $bDidSomething = true; + $cSetup->importTigerData(); +} + +if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->calculatePostcodes($aCMDResult['all']); +} + +if ($aCMDResult['index'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->index($aCMDResult['index-noanalyse']); +} + +if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createSearchIndices(); +} + +if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createCountryNames($aCMDResult); +} + +if ($aCMDResult['drop']) { + $bDidSomething = true; + $cSetup->drop($aCMDResult); +} + +// ****************************************************** +// If we did something, repeat the warnings +if (!$bDidSomething) { + showUsage($aCMDOptions, true); +} else { + echo "Summary of warnings:\n\n"; + repeatWarnings(); + echo "\n"; + info('Setup finished.'); +} diff --git a/utils/update.php b/utils/update.php index fd361eb8..176bde06 100755 --- a/utils/update.php +++ b/utils/update.php @@ -1,66 +1,42 @@ #!@PHP_BIN@ -Cq getTotalMemoryMB()) { $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } -$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; +$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port']; if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; } @@ -73,12 +49,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { } if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; + $sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; } if ($aResult['init-updates']) { // sanity check that the replication URL is correct - $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt'); + $sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt'); if ($sBaseState === false) { echo "\nCannot find state.txt file at the configured replication URL.\n"; echo "Does the URL point to a directory containing OSM update data?\n\n"; @@ -92,7 +68,7 @@ if ($aResult['init-updates']) { fail('CONST_Pyosmium_Binary not configured'); } $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary.' --help'; + $sCmd = CONST_Pyosmium_Binary . ' --help'; exec($sCmd, $aOutput, $iRet); if ($iRet != 0) { echo "Cannot execute pyosmium-get-changes.\n"; @@ -100,19 +76,23 @@ if ($aResult['init-updates']) { echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n"; fail('pyosmium-get-changes not found or not usable'); } + if (!$aResult['no-update-functions']) { - createFunctions($aCMDResult); + // instatiate setupClass to use the function therein + // instantiate Setup class + $cSetup = new SetupFunctions($aResult); + $cSetup->createFunctions(); } $sDatabaseDate = getDatabaseDate($oDB); if ($sDatabaseDate === false) { fail('Cannot determine date of database.'); } - $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60)); + $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60)); // get the appropriate state id $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url; + $sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url; exec($sCmd, $aOutput, $iRet); if ($iRet != 0 || $aOutput[0] == 'None') { fail('Error running pyosmium tools'); @@ -120,7 +100,7 @@ if ($aResult['init-updates']) { pg_query($oDB->connection, 'TRUNCATE import_status'); $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('"; - $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)'; + $sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)'; if (!pg_query($oDB->connection, $sSQL)) { fail('Could not enter sequence into database.'); } @@ -135,7 +115,7 @@ if ($aResult['check-for-updates']) { fail('Updates not set up. Please run ./utils/update.php --init-updates.'); } - system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet); + system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet); exit($iRet); } @@ -148,8 +128,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { } // Import the file - $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile; - echo $sCMD."\n"; + $sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile; + echo $sCMD . "\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { @@ -161,35 +141,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { if ($aResult['calculate-postcodes']) { info('Update postcodes centroids'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql'); + $sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql'); runSQLScript($sTemplate, true, true); } -$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc'; +$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc'; $bHaveDiff = false; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; $sContentURL = ''; if (isset($aResult['import-node']) && $aResult['import-node']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node']; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;'; } } if (isset($aResult['import-way']) && $aResult['import-way']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full'; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;'; } } if (isset($aResult['import-relation']) && $aResult['import-relation']) { if ($bUseOSMApi) { - $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full'; + $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;'; } } @@ -200,8 +180,8 @@ if ($sContentURL) { if ($bHaveDiff) { // import generated change file - $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile; - echo $sCMD."\n"; + $sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile; + echo $sCMD . "\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { fail("osm2pgsql exited with error level $iErrorLevel\n"); @@ -209,7 +189,7 @@ if ($bHaveDiff) { } if ($aResult['deduplicate']) { - $oDB =& getDB(); + $oDB = &getDB(); if (getPostgresVersion($oDB) < 9.3) { fail('ERROR: deduplicate is only currently supported in postgresql 9.3'); @@ -221,7 +201,7 @@ if ($aResult['deduplicate']) { // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates foreach ($aPartitions as $i => $sPartition) { - $sSQL = 'select count(*) from search_name_'.$sPartition; + $sSQL = 'select count(*) from search_name_' . $sPartition; $nEntries = chksql($oDB->getOne($sSQL)); if ($nEntries == 0) { unset($aPartitions[$i]); @@ -233,11 +213,14 @@ if ($aResult['deduplicate']) { $sSQL .= ' group by word_token having count(*) > 1 order by word_token'; $aDuplicateTokens = chksql($oDB->getAll($sSQL)); foreach ($aDuplicateTokens as $aToken) { - if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; - echo 'Deduping '.$aToken['word_token']."\n"; + if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') { + continue; + } + + echo 'Deduping ' . $aToken['word_token'] . "\n"; $sSQL = 'select word_id,'; $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num'; - $sSQL .= " from word where word_token = '".$aToken['word_token']; + $sSQL .= " from word where word_token = '" . $aToken['word_token']; $sSQL .= "' and class is null and type is null and country_code is null order by num desc"; $aTokenSet = chksql($oDB->getAll($sSQL)); @@ -246,34 +229,34 @@ if ($aResult['deduplicate']) { foreach ($aTokenSet as $aRemove) { $sSQL = 'update search_name set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); $sSQL = 'update search_name set'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); foreach ($aPartitions as $sPartition) { - $sSQL = 'update search_name_'.$sPartition.' set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; + $sSQL = 'update search_name_' . $sPartition . ' set'; + $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); } - $sSQL = 'delete from word where word_id = '.$aRemove['word_id']; + $sSQL = 'delete from word where word_id = ' . $aRemove['word_id']; chksql($oDB->query($sSQL)); } } @@ -281,12 +264,12 @@ if ($aResult['deduplicate']) { if ($aResult['recompute-word-counts']) { info('Recompute frequency of full-word search terms'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); + $sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql'); runSQLScript($sTemplate, true, true); } if ($aResult['index']) { - $sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']; + $sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCmd .= ' -H ' . $aDSNInfo['hostspec']; } @@ -301,13 +284,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { fail('Error: Update interval too low for download.geofabrik.de. ' . - "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); + "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); } - $sImportFile = CONST_InstallPath.'/osmosischange.osc'; - $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size; - $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; - $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; + $sImportFile = CONST_InstallPath . '/osmosischange.osc'; + $sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size; + $sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile; + $sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec']; } @@ -324,7 +307,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { exit(1); } - echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n"; + echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n"; $sBatchEnd = $aLastState['lastimportdate']; $iEndSequence = $aLastState['sequence_id']; @@ -347,23 +330,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (file_exists($sImportFile)) { unlink($sImportFile); } - exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult); + exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult); if ($iResult == 3) { - echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n"; + echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n"; sleep(CONST_Replication_Recheck_Interval); } elseif ($iResult != 0) { echo 'ERROR: updates failed.'; exit($iResult); } else { - $iEndSequence = (int)$aOutput[0]; + $iEndSequence = (int) $aOutput[0]; } } while ($iResult); // get the newest object from the diff file $sBatchEnd = 0; $iRet = 0; - exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet); + exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet); if ($iRet == 5) { echo "Diff file is empty. skipping import.\n"; if (!$aResult['import-osmosis-all']) { @@ -379,7 +362,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // Import the file $fCMDStartTime = time(); - echo $sCMDImport."\n"; + echo $sCMDImport . "\n"; unset($sJunk); $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv); if ($iErrorLevel) { @@ -392,8 +375,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; - $sSQL .= date('Y-m-d H:i:s')."','import')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; + $sSQL .= date('Y-m-d H:i:s') . "','import')"; var_Dump($sSQL); chksql($oDB->query($sSQL)); @@ -401,7 +384,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence"; var_Dump($sSQL); chksql($oDB->query($sSQL)); - echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; + echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; } // Index file @@ -419,18 +402,21 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; - $sSQL .= date('Y-m-d H:i:s')."','index')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; + $sSQL .= date('Y-m-d H:i:s') . "','index')"; var_Dump($sSQL); $oDB->query($sSQL); - echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; + echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; $sSQL = 'update import_status set indexed = true'; $oDB->query($sSQL); } $fDuration = time() - $fStartTime; - echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n"; - if (!$aResult['import-osmosis-all']) exit(0); + echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n"; + if (!$aResult['import-osmosis-all']) { + exit(0); + } + } }