X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/02bd322c1dc185a6eef072de7398c190fe7a2e0c..5c187e131a372d0cbdbba3372668862e3f1c47f9:/utils/setup.php?ds=inline diff --git a/utils/setup.php b/utils/setup.php index 60fe5dac..992034c7 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -37,7 +37,6 @@ array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'), array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); @@ -94,7 +93,7 @@ { echo "Create DB\n"; $bDidSomething = true; - $oDB =& DB::connect(CONST_Database_DSN, false); + $oDB = DB::connect(CONST_Database_DSN, false); if (!PEAR::isError($oDB)) { fail('database already exists ('.CONST_Database_DSN.')'); @@ -124,11 +123,8 @@ // For extratags and namedetails the hstore_to_json converter is // needed which is only available from Postgresql 9.3+. For older // versions add a dummy function that returns nothing. - $iNumFunc = $oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'"); - if (PEAR::isError($iNumFunc)) - { - fail("Cannot query stored procedures.", $iNumFunc); - } + $iNumFunc = chksql($oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); + if ($iNumFunc == 0) { pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); @@ -156,9 +152,10 @@ { echo "WARNING: external UK postcode table not found.\n"; } - pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql'); - pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql'); - pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); + if (CONST_Use_Extra_US_Postcodes) + { + pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); + } if ($aCMDResult['no-partitions']) { @@ -204,11 +201,10 @@ passthruCheckReturn($osm2pgsql); $oDB =& getDB(); - $x = $oDB->getRow('select * from place limit 1'); - if (PEAR::isError($x)) { - fail($x->getMessage()); + if (!chksql($oDB->getRow('select * from place limit 1'))) + { + fail('No Data'); } - if (!$x) fail('No Data'); } if ($aCMDResult['create-functions'] || $aCMDResult['all']) @@ -216,12 +212,7 @@ echo "Functions\n"; $bDidSomething = true; if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built"); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate); - if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate); - if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate); - if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); - pgsqlRunScript($sTemplate); + create_sql_functions($aCMDResult); } if ($aCMDResult['create-tables'] || $aCMDResult['all']) @@ -247,24 +238,13 @@ // re-run the functions echo "Functions\n"; - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', - CONST_InstallPath.'/module', $sTemplate); - pgsqlRunScript($sTemplate); + create_sql_functions($aCMDResult); } if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { echo "Partition Tables\n"; $bDidSomething = true; - $oDB =& getDB(); - $sSQL = 'select distinct partition from country_name'; - $aPartitions = $oDB->getCol($sSQL); - if (PEAR::isError($aPartitions)) - { - fail($aPartitions->getMessage()); - } - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); $sTemplate = replace_tablespace('{ts:address-data}', @@ -279,18 +259,8 @@ CONST_Tablespace_Aux_Data, $sTemplate); $sTemplate = replace_tablespace('{ts:aux-index}', CONST_Tablespace_Aux_Index, $sTemplate); - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach($aMatches as $aMatch) - { - $sResult = ''; - foreach($aPartitions as $sPartitionName) - { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - pgsqlRunScript($sTemplate); + pgsqlRunPartitionScript($sTemplate); } @@ -298,28 +268,10 @@ { echo "Partition Functions\n"; $bDidSomething = true; - $oDB =& getDB(); - $sSQL = 'select distinct partition from country_name'; - $aPartitions = $oDB->getCol($sSQL); - if (PEAR::isError($aPartitions)) - { - fail($aPartitions->getMessage()); - } - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach($aMatches as $aMatch) - { - $sResult = ''; - foreach($aPartitions as $sPartitionName) - { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - pgsqlRunScript($sTemplate); + pgsqlRunPartitionScript($sTemplate); } if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) @@ -360,6 +312,8 @@ echo '.'; if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection)); echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($oDB->connection)); + echo '.'; if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection)); echo '.'; if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection)); @@ -376,11 +330,7 @@ echo '.'; $sSQL = 'select distinct partition from country_name'; - $aPartitions = $oDB->getCol($sSQL); - if (PEAR::isError($aPartitions)) - { - fail($aPartitions->getMessage()); - } + $aPartitions = chksql($oDB->getCol($sSQL)); if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; foreach($aPartitions as $sPartition) { @@ -401,20 +351,30 @@ echo "Load Data\n"; $aDBInstances = array(); - for($i = 0; $i < $iInstances; $i++) + $iLoadThreads = max(1, $iInstances - 1); + for($i = 0; $i < $iLoadThreads; $i++) { $aDBInstances[$i] =& getDB(true); $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, '; $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, '; - $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i; + $sSQL .= 'geometry) select * from place where osm_id % '.$iLoadThreads.' = '.$i; + $sSQL .= " and not (class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString')"; if ($aCMDResult['verbose']) echo "$sSQL\n"; if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection)); } + // last thread for interpolation lines + $aDBInstances[$iLoadThreads] =& getDB(true); + $sSQL = 'select insert_osmline (osm_id, housenumber, street, addr_place, postcode, country_code, '; + $sSQL .= 'geometry) from place where '; + $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; + if ($aCMDResult['verbose']) echo "$sSQL\n"; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection)); + $bAnyBusy = true; while($bAnyBusy) { $bAnyBusy = false; - for($i = 0; $i < $iInstances; $i++) + for($i = 0; $i <= $iLoadThreads; $i++) { if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; } @@ -509,10 +469,13 @@ $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; - $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',"; - $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode"; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); + if (CONST_Use_Extra_US_Postcodes) + { + $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; + $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',"; + $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode"; + if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); + } } if ($aCMDResult['osmosis-init'] || ($aCMDResult['all'] && !$aCMDResult['drop'])) // no use doing osmosis-init when dropping update tables @@ -625,14 +588,6 @@ { echo "Search indices\n"; $bDidSomething = true; - $oDB =& getDB(); - $sSQL = 'select distinct partition from country_name'; - $aPartitions = $oDB->getCol($sSQL); - if (PEAR::isError($aPartitions)) - { - fail($aPartitions->getMessage()); - } - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); $sTemplate = replace_tablespace('{ts:address-index}', @@ -641,51 +596,10 @@ CONST_Tablespace_Search_Index, $sTemplate); $sTemplate = replace_tablespace('{ts:aux-index}', CONST_Tablespace_Aux_Index, $sTemplate); - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach($aMatches as $aMatch) - { - $sResult = ''; - foreach($aPartitions as $sPartitionName) - { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } pgsqlRunScript($sTemplate); } - if (isset($aCMDResult['create-website'])) - { - $bDidSomething = true; - $sTargetDir = $aCMDResult['create-website']; - if (!is_dir($sTargetDir)) - { - echo "You must create the website directory before calling this function.\n"; - fail("Target directory does not exist."); - } - - @symlink(CONST_InstallPath.'/website/details.php', $sTargetDir.'/details.php'); - @symlink(CONST_InstallPath.'/website/reverse.php', $sTargetDir.'/reverse.php'); - @symlink(CONST_InstallPath.'/website/search.php', $sTargetDir.'/search.php'); - @symlink(CONST_InstallPath.'/website/search.php', $sTargetDir.'/index.php'); - @symlink(CONST_InstallPath.'/website/lookup.php', $sTargetDir.'/lookup.php'); - @symlink(CONST_InstallPath.'/website/deletable.php', $sTargetDir.'/deletable.php'); - @symlink(CONST_InstallPath.'/website/polygons.php', $sTargetDir.'/polygons.php'); - @symlink(CONST_InstallPath.'/website/status.php', $sTargetDir.'/status.php'); - @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images'); - @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js'); - @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css'); - echo "Symlinks created\n"; - - $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js'); - if (!$sTestFile) - { - echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n"; - echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n"; - } - } - if ($aCMDResult['drop']) { // The implementation is potentially a bit dangerous because it uses @@ -715,11 +629,8 @@ $oDB =& getDB(); $aDropTables = array(); - $aHaveTables = $oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"); - if (PEAR::isError($aHaveTables)) - { - fail($aPartitions->getMessage()); - } + $aHaveTables = chksql($oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); + foreach($aHaveTables as $sTable) { $bFound = false; @@ -848,6 +759,29 @@ } } + function pgsqlRunPartitionScript($sTemplate) + { + global $aCMDResult; + $oDB =& getDB(); + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($oDB->getCol($sSQL)); + if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; + + preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); + foreach($aMatches as $aMatch) + { + $sResult = ''; + foreach($aPartitions as $sPartitionName) + { + $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); + } + $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); + } + + pgsqlRunScript($sTemplate); + } + function pgsqlRunRestoreData($sDumpFile) { // Convert database DSN to psql parameters @@ -922,3 +856,31 @@ return $sSql; } + function create_sql_functions($aCMDResult) + { + $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); + $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate); + if ($aCMDResult['enable-diff-updates']) + { + $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); + } + if ($aCMDResult['enable-debug-statements']) + { + $sTemplate = str_replace('--DEBUG:', '', $sTemplate); + } + if (CONST_Limit_Reindexing) + { + $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); + } + if (!CONST_Use_US_Tiger_Data) + { + $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); + } + if (!CONST_Use_Aux_Location_data) + { + $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); + } + pgsqlRunScript($sTemplate); + + } +