require_once(CONST_BasePath.'/lib/init-cmd.php');
ini_set('memory_limit', '800M');
+# (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
+
$aCMDOptions
= array(
- "Create and setup nominatim search system",
+ 'Create and setup nominatim search system',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
+ array('module-path', '', 0, 1, 1, 1, 'string', 'Directory on Postgres server containing Nominatim module'),
array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
- array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
+ array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
+ array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'),
array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
);
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
}
}
+// by default, use all but one processor, but never more than 15.
+$iInstances = isset($aCMDResult['threads'])
+ ? $aCMDResult['threads']
+ : (min(16, getProcessorCount()) - 1);
-// This is a pretty hard core default - the number of processors in the box - 1
-$iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
if ($iInstances < 1) {
$iInstances = 1;
- echo "WARNING: resetting threads to $iInstances\n";
-}
-if ($iInstances > getProcessorCount()) {
- $iInstances = getProcessorCount();
- echo "WARNING: resetting threads to $iInstances\n";
+ warn("resetting threads to $iInstances");
}
// Assume we can steal all the cache memory in the box (unless told otherwise)
$iCacheMemory = getCacheMemoryMB();
}
+$sModulePath = CONST_InstallPath . '/module';
+if (isset($aCMDResult['module-path'])) {
+ $sModulePath = $aCMDResult['module-path'];
+ echo 'module path: ' . $sModulePath . '\n';
+}
+
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
- echo "Create DB\n";
+ info('Create DB');
$bDidSomething = true;
$oDB = DB::connect(CONST_Database_DSN, false);
if (!PEAR::isError($oDB)) {
fail('database already exists ('.CONST_Database_DSN.')');
}
- passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
+
+ $sCreateDBCmd = 'createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database'];
+ if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+ $sCreateDBCmd .= ' -U ' . $aDSNInfo['username'];
+ }
+ if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+ $sCreateDBCmd .= ' -h ' . $aDSNInfo['hostspec'];
+ }
+
+ $aProcEnv = null;
+ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
+ $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
+ }
+
+ $result = runWithEnv($sCreateDBCmd, $aProcEnv);
+ if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
}
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
- echo "Setup DB\n";
+ info('Setup DB');
$bDidSomething = true;
- // TODO: path detection, detection memory, etc.
$oDB =& getDB();
echo 'Postgres version found: '.$fPostgresVersion."\n";
if ($fPostgresVersion < 9.1) {
- fail("Minimum supported version of Postgresql is 9.1.");
+ fail('Minimum supported version of Postgresql is 9.1.');
}
pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
if ($iNumFunc == 0) {
pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
- echo "WARNING: Postgresql is too old. extratags and namedetails API not available.";
+ warn('Postgresql is too old. extratags and namedetails API not available.');
}
$fPostgisVersion = getPostgisVersion($oDB);
echo 'Postgis version found: '.$fPostgisVersion."\n";
if ($fPostgisVersion < 2.1) {
- // Function was renamed in 2.1 and throws an annoying deprecation warning
+ // Functions were renamed in 2.1 and throw an annoying deprecation warning
pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
+ pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint');
+ }
+ if ($fPostgisVersion < 2.2) {
+ pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
+ }
+
+ $i = chksql($oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
+ if ($i == 0) {
+ echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
+ echo "\n createuser ".CONST_Database_Web_User."\n\n";
+ exit(1);
+ }
+
+ // Try accessing the C module, so we know early if something is wrong
+ // and can simply error out.
+ $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
+ $sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
+ $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
+ $oResult = $oDB->query($sSQL);
+
+ if (PEAR::isError($oResult)) {
+ echo "\nERROR: Failed to load nominatim module. Reason:\n";
+ echo $oResult->userinfo."\n\n";
+ exit(1);
+ }
+
+ if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
+ echo 'Error: you need to download the country_osm_grid first:';
+ echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
+ exit(1);
}
pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
- pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
+ pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz');
pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
} else {
- echo "WARNING: external UK postcode table not found.\n";
+ warn('external UK postcode table not found.');
}
if (CONST_Use_Extra_US_Postcodes) {
pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
// is only defined in the subsequently called create_tables.
// Create dummies here that will be overwritten by the proper
// versions in create-tables.
- pgsqlRunScript('CREATE TABLE place_boundingbox ()');
- pgsqlRunScript('create type wikipedia_article_match as ()');
+ pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()');
+ pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false);
}
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
- echo "Import\n";
+ info('Import data');
$bDidSomething = true;
$osm2pgsql = CONST_Osm2pgsql_Binary;
if (!file_exists($osm2pgsql)) {
- echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
+ echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
+ echo "Normally you should not need to set this manually.\n";
fail("osm2pgsql not found in '$osm2pgsql'");
}
- if (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
+ if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
}
if (CONST_Tablespace_Osm2pgsql_Data)
$osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
$osm2pgsql .= ' -C '.$iCacheMemory;
$osm2pgsql .= ' -P '.$aDSNInfo['port'];
+ if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+ $osm2pgsql .= ' -U ' . $aDSNInfo['username'];
+ }
+ if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+ $osm2pgsql .= ' -H ' . $aDSNInfo['hostspec'];
+ }
+
+ $aProcEnv = null;
+ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
+ $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
+ }
+
$osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
- passthruCheckReturn($osm2pgsql);
+ runWithEnv($osm2pgsql, $aProcEnv);
$oDB =& getDB();
- if (!chksql($oDB->getRow('select * from place limit 1'))) {
+ if (!$aCMDResult['ignore-errors'] && !chksql($oDB->getRow('select * from place limit 1'))) {
fail('No Data');
}
}
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
- echo "Functions\n";
+ info('Create Functions');
$bDidSomething = true;
- if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
create_sql_functions($aCMDResult);
}
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
+ info('Create Tables');
$bDidSomething = true;
- echo "Tables\n";
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
- $sTemplate = replace_tablespace('{ts:address-data}',
- CONST_Tablespace_Address_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:address-index}',
- CONST_Tablespace_Address_Index, $sTemplate);
- $sTemplate = replace_tablespace('{ts:search-data}',
- CONST_Tablespace_Search_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:search-index}',
- CONST_Tablespace_Search_Index, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-data}',
- CONST_Tablespace_Aux_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-index}',
- CONST_Tablespace_Aux_Index, $sTemplate);
+ $sTemplate = replace_tablespace(
+ '{ts:address-data}',
+ CONST_Tablespace_Address_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:address-index}',
+ CONST_Tablespace_Address_Index,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:search-data}',
+ CONST_Tablespace_Search_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:search-index}',
+ CONST_Tablespace_Search_Index,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:aux-data}',
+ CONST_Tablespace_Aux_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:aux-index}',
+ CONST_Tablespace_Aux_Index,
+ $sTemplate
+ );
pgsqlRunScript($sTemplate, false);
// re-run the functions
- echo "Functions\n";
+ info('Recreate Functions');
create_sql_functions($aCMDResult);
}
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
- echo "Partition Tables\n";
+ info('Create Partition Tables');
$bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
- $sTemplate = replace_tablespace('{ts:address-data}',
- CONST_Tablespace_Address_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:address-index}',
- CONST_Tablespace_Address_Index, $sTemplate);
- $sTemplate = replace_tablespace('{ts:search-data}',
- CONST_Tablespace_Search_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:search-index}',
- CONST_Tablespace_Search_Index, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-data}',
- CONST_Tablespace_Aux_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-index}',
- CONST_Tablespace_Aux_Index, $sTemplate);
+ $sTemplate = replace_tablespace(
+ '{ts:address-data}',
+ CONST_Tablespace_Address_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:address-index}',
+ CONST_Tablespace_Address_Index,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:search-data}',
+ CONST_Tablespace_Search_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:search-index}',
+ CONST_Tablespace_Search_Index,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:aux-data}',
+ CONST_Tablespace_Aux_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:aux-index}',
+ CONST_Tablespace_Aux_Index,
+ $sTemplate
+ );
pgsqlRunPartitionScript($sTemplate);
}
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
- echo "Partition Functions\n";
+ info('Create Partition Functions');
$bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$bDidSomething = true;
- $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
- $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
+ $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
+ $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
if (file_exists($sWikiArticlesFile)) {
- echo "Importing wikipedia articles...";
+ info('Importing wikipedia articles');
pgsqlRunDropAndRestore($sWikiArticlesFile);
- echo "...done\n";
} else {
- echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
+ warn('wikipedia article dump file not found - places will have default importance');
}
if (file_exists($sWikiRedirectsFile)) {
- echo "Importing wikipedia redirects...";
+ info('Importing wikipedia redirects');
pgsqlRunDropAndRestore($sWikiRedirectsFile);
- echo "...done\n";
} else {
- echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
+ warn('wikipedia redirect dump file not found - some place importance values may be missing');
}
}
if ($aCMDResult['load-data'] || $aCMDResult['all']) {
- echo "Drop old Data\n";
+ info('Drop old Data');
$bDidSomething = true;
$oDB =& getDB();
}
// used by getorcreate_word_id to ignore frequent partial words
- if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
+ $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
+ $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
+ if (!pg_query($oDB->connection, $sSQL)) {
+ fail(pg_last_error($oDB->connection));
+ }
echo ".\n";
// pre-create the word list
if (!$aCMDResult['disable-token-precalc']) {
- echo "Loading word list\n";
+ info('Loading word list');
pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
}
- echo "Load Data\n";
+ info('Load Data');
+ $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
+
$aDBInstances = array();
$iLoadThreads = max(1, $iInstances - 1);
for ($i = 0; $i < $iLoadThreads; $i++) {
$aDBInstances[$i] =& getDB(true);
- $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
- $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
- $sSQL .= 'geometry) select * from place where osm_id % '.$iLoadThreads.' = '.$i;
- $sSQL .= " and not (class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString')";
+ $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
+ $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
+ $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')";
+ $sSQL .= ' and ST_IsValid(geometry)';
if ($aCMDResult['verbose']) echo "$sSQL\n";
- if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
+ if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
+ fail(pg_last_error($aDBInstances[$i]->connection));
+ }
}
// last thread for interpolation lines
$aDBInstances[$iLoadThreads] =& getDB(true);
- $sSQL = 'select insert_osmline (osm_id, housenumber, street, addr_place, postcode, country_code, ';
- $sSQL .= 'geometry) from place where ';
+ $sSQL = 'insert into location_property_osmline';
+ $sSQL .= ' (osm_id, address, linegeo)';
+ $sSQL .= ' SELECT osm_id, address, geometry from place where ';
$sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
if ($aCMDResult['verbose']) echo "$sSQL\n";
- if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
-
- $bAnyBusy = true;
- while ($bAnyBusy) {
- $bAnyBusy = false;
- for ($i = 0; $i <= $iLoadThreads; $i++) {
- if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
+ if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
+ fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
+ }
+
+ $bFailed = false;
+ for ($i = 0; $i <= $iLoadThreads; $i++) {
+ while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) {
+ $resultStatus = pg_result_status($hPGresult);
+ // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
+ // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,
+ // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR
+ echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n";
+ if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) {
+ $resultError = pg_result_error($hPGresult);
+ echo '-- error text ' . $i . ': ' . $resultError . "\n";
+ $bFailed = true;
+ }
}
- sleep(1);
- echo '.';
+ }
+ if ($bFailed) {
+ fail('SQL errors loading placex and/or location_property_osmline tables');
}
echo "\n";
- echo "Reanalysing database...\n";
+ info('Reanalysing database');
pgsqlRunScript('ANALYSE');
+
+ $sDatabaseDate = getDatabaseDate($oDB);
+ pg_query($oDB->connection, 'TRUNCATE import_status');
+ if ($sDatabaseDate === false) {
+ warn('could not determine database date.');
+ } else {
+ $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
+ pg_query($oDB->connection, $sSQL);
+ echo "Latest data imported from $sDatabaseDate.\n";
+ }
}
if ($aCMDResult['import-tiger-data']) {
+ info('Import Tiger data');
$bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-data}',
- CONST_Tablespace_Aux_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-index}',
- CONST_Tablespace_Aux_Index, $sTemplate);
+ $sTemplate = replace_tablespace(
+ '{ts:aux-data}',
+ CONST_Tablespace_Aux_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:aux-index}',
+ CONST_Tablespace_Aux_Index,
+ $sTemplate
+ );
pgsqlRunScript($sTemplate, false);
$aDBInstances = array();
foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {
echo $sFile.': ';
- $hFile = fopen($sFile, "r");
+ $hFile = fopen($sFile, 'r');
$sSQL = fgets($hFile, 100000);
$iLines = 0;
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
$iLines++;
if ($iLines == 1000) {
- echo ".";
+ echo '.';
$iLines = 0;
}
}
echo "\n";
}
- echo "Creating indexes\n";
+ info('Creating indexes on Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-data}',
- CONST_Tablespace_Aux_Data, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-index}',
- CONST_Tablespace_Aux_Index, $sTemplate);
+ $sTemplate = replace_tablespace(
+ '{ts:aux-data}',
+ CONST_Tablespace_Aux_Data,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:aux-index}',
+ CONST_Tablespace_Aux_Index,
+ $sTemplate
+ );
pgsqlRunScript($sTemplate, false);
}
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
+ info('Calculate Postcodes');
$bDidSomething = true;
$oDB =& getDB();
- if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
- $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
- $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
- $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
- $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
- $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
- if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
+ if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) {
+ fail(pg_last_error($oDB->connection));
+ }
+
+ $sSQL = 'INSERT INTO location_postcode';
+ $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+ $sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
+ $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,";
+ $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))';
+ $sSQL .= ' FROM placex';
+ $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
+ $sSQL .= ' AND geometry IS NOT null';
+ $sSQL .= ' GROUP BY country_code, pc';
+
+ if (!pg_query($oDB->connection, $sSQL)) {
+ fail(pg_last_error($oDB->connection));
+ }
if (CONST_Use_Extra_US_Postcodes) {
- $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
- $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
- $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
+ // only add postcodes that are not yet available in OSM
+ $sSQL = 'INSERT INTO location_postcode';
+ $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+ $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
+ $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)';
+ $sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
+ $sSQL .= ' (SELECT postcode FROM location_postcode';
+ $sSQL .= " WHERE country_code = 'us')";
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
}
-}
-if ($aCMDResult['osmosis-init'] || ($aCMDResult['all'] && !$aCMDResult['drop'])) { // no use doing osmosis-init when dropping update tables
- $bDidSomething = true;
- $oDB =& getDB();
+ // add missing postcodes for GB (if available)
+ $sSQL = 'INSERT INTO location_postcode';
+ $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+ $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
+ $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
+ $sSQL .= ' (SELECT postcode FROM location_postcode';
+ $sSQL .= " WHERE country_code = 'gb')";
+ if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
- if (!file_exists(CONST_Osmosis_Binary)) {
- echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
- if (!$aCMDResult['all']) {
- fail("osmosis not found in '".CONST_Osmosis_Binary."'");
- }
- } else {
- if (file_exists(CONST_InstallPath.'/settings/configuration.txt')) {
- echo "settings/configuration.txt already exists\n";
- } else {
- passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_InstallPath.'/settings');
- // update osmosis configuration.txt with our settings
- passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_InstallPath.'/settings/configuration.txt');
- passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_InstallPath.'/settings/configuration.txt');
+ if (!$aCMDResult['all']) {
+ $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
+ $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
+ if (!pg_query($oDB->connection, $sSQL)) {
+ fail(pg_last_error($oDB->connection));
}
+ }
+ $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
+ $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
- // Find the last node in the DB
- $iLastOSMID = $oDB->getOne("select max(osm_id) from place where osm_type = 'N'");
-
- // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
- $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
- $sLastNodeXML = file_get_contents($sLastNodeURL);
- preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
- $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
-
- // Search for the correct state file - uses file timestamps so need to sort by date descending
- $sRepURL = CONST_Replication_Url."/";
- $sRep = file_get_contents($sRepURL."?C=M;O=D;F=1");
- // download.geofabrik.de: <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53 </td>
- // planet.openstreetmap.org: <a href="273/">273/</a> 2013-03-11 07:41 -
- preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>\s*([-0-9a-zA-Z]+ [0-9]{2}:[0-9]{2})#', $sRep, $aRepMatches, PREG_SET_ORDER);
- if ($aRepMatches) {
- $aPrevRepMatch = false;
- foreach ($aRepMatches as $aRepMatch) {
- if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
- $aPrevRepMatch = $aRepMatch;
- }
- if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
-
- $sRepURL .= $aRepMatch[1];
- $sRep = file_get_contents($sRepURL."?C=M;O=D;F=1");
- preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>\s*([-0-9a-zA-Z]+ [0-9]{2}:[0-9]{2})#', $sRep, $aRepMatches, PREG_SET_ORDER);
- $aPrevRepMatch = false;
- foreach ($aRepMatches as $aRepMatch) {
- if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
- $aPrevRepMatch = $aRepMatch;
- }
- if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
-
- $sRepURL .= $aRepMatch[1];
- $sRep = file_get_contents($sRepURL."?C=M;O=D;F=1");
- preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>\s*([-0-9a-zA-Z]+ [0-9]{2}:[0-9]{2})#', $sRep, $aRepMatches, PREG_SET_ORDER);
- $aPrevRepMatch = false;
- foreach ($aRepMatches as $aRepMatch) {
- if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
- $aPrevRepMatch = $aRepMatch;
- }
- if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
-
- $sRepURL .= $aRepMatch[1].'.state.txt';
- echo "Getting state file: $sRepURL\n";
- $sStateFile = file_get_contents($sRepURL);
- if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
- file_put_contents(CONST_InstallPath.'/settings/state.txt', $sStateFile);
- echo "Updating DB status\n";
- pg_query($oDB->connection, 'TRUNCATE import_status');
- $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
- pg_query($oDB->connection, $sSQL);
- } else {
- if (!$aCMDResult['all']) {
- fail("Cannot read state file directory.");
- }
- }
+ if (!pg_query($oDB->connection, $sSQL)) {
+ fail(pg_last_error($oDB->connection));
}
}
+if ($aCMDResult['osmosis-init']) {
+ $bDidSomething = true;
+ echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n";
+}
+
if ($aCMDResult['index'] || $aCMDResult['all']) {
$bDidSomething = true;
$sOutputFile = '';
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
- passthruCheckReturn($sBaseCmd.' -R 4');
+ if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+ $sBaseCmd .= ' -H ' . $aDSNInfo['hostspec'];
+ }
+ if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+ $sBaseCmd .= ' -U ' . $aDSNInfo['username'];
+ }
+ $aProcEnv = null;
+ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
+ $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
+ }
+
+ info('Index ranks 0 - 4');
+ $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv);
+ if ($iStatus != 0) {
+ fail('error status ' . $iStatus . ' running nominatim!');
+ }
if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
- passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
+ info('Index ranks 5 - 25');
+ $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv);
+ if ($iStatus != 0) {
+ fail('error status ' . $iStatus . ' running nominatim!');
+ }
if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
- passthruCheckReturn($sBaseCmd.' -r 26');
+ info('Index ranks 26 - 30');
+ $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv);
+ if ($iStatus != 0) {
+ fail('error status ' . $iStatus . ' running nominatim!');
+ }
+
+ info('Index postcodes');
+ $oDB =& getDB();
+ $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
+ if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
}
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
- echo "Search indices\n";
+ info('Create Search indices');
$bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
- $sTemplate = replace_tablespace('{ts:address-index}',
- CONST_Tablespace_Address_Index, $sTemplate);
- $sTemplate = replace_tablespace('{ts:search-index}',
- CONST_Tablespace_Search_Index, $sTemplate);
- $sTemplate = replace_tablespace('{ts:aux-index}',
- CONST_Tablespace_Aux_Index, $sTemplate);
+ $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
+ $sTemplate = replace_tablespace(
+ '{ts:address-index}',
+ CONST_Tablespace_Address_Index,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:search-index}',
+ CONST_Tablespace_Search_Index,
+ $sTemplate
+ );
+ $sTemplate = replace_tablespace(
+ '{ts:aux-index}',
+ CONST_Tablespace_Aux_Index,
+ $sTemplate
+ );
pgsqlRunScript($sTemplate);
}
+if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
+ info('Create search index for default country names');
+ $bDidSomething = true;
+
+ pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')");
+ pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')");
+ pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
+ pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
+
+ $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
+ if (CONST_Languages) {
+ $sSQL .= 'in ';
+ $sDelim = '(';
+ foreach (explode(',', CONST_Languages) as $sLang) {
+ $sSQL .= $sDelim."'name:$sLang'";
+ $sDelim = ',';
+ }
+ $sSQL .= ')';
+ } else {
+ // all include all simple name tags
+ $sSQL .= "like 'name:%'";
+ }
+ $sSQL .= ') v';
+ pgsqlRunScript($sSQL);
+}
+
if ($aCMDResult['drop']) {
+ info('Drop tables only required for updates');
// The implementation is potentially a bit dangerous because it uses
// a positive selection of tables to keep, and deletes everything else.
// Including any tables that the unsuspecting user might have manually
// tables we want to keep. everything else goes.
$aKeepTables = array(
- "*columns",
- "import_polygon_*",
- "import_status",
- "place_addressline",
- "location_property*",
- "placex",
- "search_name",
- "seq_*",
- "word",
- "query_log",
- "new_query_log",
- "gb_postcode",
- "spatial_ref_sys",
- "country_name",
- "place_classtype_*"
+ '*columns',
+ 'import_polygon_*',
+ 'import_status',
+ 'place_addressline',
+ 'location_postcode',
+ 'location_property*',
+ 'placex',
+ 'search_name',
+ 'seq_*',
+ 'word',
+ 'query_log',
+ 'new_query_log',
+ 'spatial_ref_sys',
+ 'country_name',
+ 'place_classtype_*'
);
$oDB =& getDB();
// been deleted already by CASCADE
}
- if (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
- if ($aCMDResult['verbose']) echo "deleting ".CONST_Osm2pgsql_Flatnode_File."\n";
+ if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
+ if ($aCMDResult['verbose']) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
unlink(CONST_Osm2pgsql_Flatnode_File);
}
}
if (!$bDidSomething) {
showUsage($aCMDOptions, true);
} else {
- echo "Setup finished.\n";
+ echo "Summary of warnings:\n\n";
+ repeatWarnings();
+ echo "\n";
+ info('Setup finished.');
}
+
function pgsqlRunScriptFile($sFilename)
{
+ global $aCMDResult;
if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
// Convert database DSN to psql parameters
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
$sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+ if (!$aCMDResult['verbose']) {
+ $sCMD .= ' -q';
+ }
+ if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+ $sCMD .= ' -h ' . $aDSNInfo['hostspec'];
+ }
+ if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+ $sCMD .= ' -U ' . $aDSNInfo['username'];
+ }
+ $aProcEnv = null;
+ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
+ $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
+ }
$ahGzipPipes = null;
if (preg_match('/\\.gz$/', $sFilename)) {
2 => array('file', '/dev/null', 'a')
);
$ahPipes = null;
- $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
+ $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
if (!is_resource($hProcess)) fail('unable to start pgsql');
-
// TODO: error checking
while (!feof($ahPipes[1])) {
echo fread($ahPipes[1], 4096);
fclose($ahGzipPipes[1]);
proc_close($hGzipProcess);
}
-
}
function pgsqlRunScript($sScript, $bfatal = true)
{
global $aCMDResult;
- // Convert database DSN to psql parameters
- $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
- if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
- $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
- if ($bfatal && !$aCMDResult['ignore-errors'])
- $sCMD .= ' -v ON_ERROR_STOP=1';
- $aDescriptors = array(
- 0 => array('pipe', 'r'),
- 1 => STDOUT,
- 2 => STDERR
- );
- $ahPipes = null;
- $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
- if (!is_resource($hProcess)) fail('unable to start pgsql');
-
- while (strlen($sScript)) {
- $written = fwrite($ahPipes[0], $sScript);
- if ($written <= 0) break;
- $sScript = substr($sScript, $written);
- }
- fclose($ahPipes[0]);
- $iReturn = proc_close($hProcess);
- if ($bfatal && $iReturn > 0) {
- fail("pgsql returned with error code ($iReturn)");
- }
+ runSQLScript(
+ $sScript,
+ $bfatal,
+ $aCMDResult['verbose'],
+ $aCMDResult['ignore-errors']
+ );
}
function pgsqlRunPartitionScript($sTemplate)
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
$sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
-
- $aDescriptors = array(
- 0 => array('pipe', 'r'),
- 1 => array('pipe', 'w'),
- 2 => array('file', '/dev/null', 'a')
- );
- $ahPipes = null;
- $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
- if (!is_resource($hProcess)) fail('unable to start pg_restore');
-
- fclose($ahPipes[0]);
-
- // TODO: error checking
- while (!feof($ahPipes[1])) {
- echo fread($ahPipes[1], 4096);
+ if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+ $sCMD .= ' -h ' . $aDSNInfo['hostspec'];
+ }
+ if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+ $sCMD .= ' -U ' . $aDSNInfo['username'];
+ }
+ $aProcEnv = null;
+ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
+ $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
}
- fclose($ahPipes[1]);
- $iReturn = proc_close($hProcess);
+ $iReturn = runWithEnv($sCMD, $aProcEnv);
}
-function passthruCheckReturn($cmd)
+function passthruCheckReturn($sCmd)
{
- $result = -1;
- passthru($cmd, $result);
- if ($result != 0) fail('Error executing external command: '.$cmd);
+ $iResult = -1;
+ passthru($sCmd, $iResult);
}
function replace_tablespace($sTemplate, $sTablespace, $sSql)
{
if ($sTablespace) {
- $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"',
- $sSql);
+ $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else {
$sSql = str_replace($sTemplate, '', $sSql);
}
function create_sql_functions($aCMDResult)
{
+ global $sModulePath;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
- $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
+ $sTemplate = str_replace('{modulepath}', $sModulePath, $sTemplate);
if ($aCMDResult['enable-diff-updates']) {
$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
}
$sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
}
pgsqlRunScript($sTemplate);
-
}
-
projects / nominatim.git / blobdiff