From: Sarah Hoffmann Date: Thu, 3 Jan 2013 21:13:41 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~652 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/bcbba1cb5a4e5a1df3b8f5c79e2a988221adf867?ds=inline;hp=-c Merge remote-tracking branch 'upstream/master' --- bcbba1cb5a4e5a1df3b8f5c79e2a988221adf867 diff --combined settings/settings.php index 5231692f,6f6f843b..33bdad51 --- a/settings/settings.php +++ b/settings/settings.php @@@ -7,8 -7,11 +7,11 @@@ @define('CONST_Database_DSN', 'pgsql://@/nominatim'); @define('CONST_Max_Word_Frequency', '50000'); + // Software versions + @define('CONST_Postgresql_Version', '9.1'); // values: 8.3, 8.4, 9.0, 9.1, 9.2 + @define('CONST_Postgis_Version', '1.5'); // values: 1.5, 2.0 + // Paths - @define('CONST_Postgresql_Version', '9.1'); @define('CONST_Path_Postgresql_Contrib', '/usr/share/postgresql/'.CONST_Postgresql_Version.'/contrib'); @define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-1.5'); @define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql'); @@@ -38,14 -41,12 +41,14 @@@ // Website settings @define('CONST_NoAccessControl', true); - @define('CONST_ClosedForIndexing', false); - @define('CONST_ClosedForIndexingExceptionIPs', ''); @define('CONST_BlockedIPs', ''); + @define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks'); + @define('CONST_WhitelistedIPs', ''); + @define('CONST_BlockedUserAgents', ''); + @define('CONST_BlockReverseMaxLoad', 15); @define('CONST_BulkUserIPs', ''); - @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); + @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/'); @define('CONST_Tile_Default', 'Mapnik'); @define('CONST_Default_Language', 'xx'); @@@ -59,6 -60,7 +62,7 @@@ @define('CONST_Suggestions_Enabled', false); @define('CONST_Search_TryDroppedAddressTerms', false); + @define('CONST_Search_NameOnlySearchFrequencyThreshold', false); // Set to zero to disable polygon output @define('CONST_PolygonOutput_MaximumTypes', 1); diff --combined utils/setup.php index 33de5d85,ff890af9..45b40fed --- a/utils/setup.php +++ b/utils/setup.php @@@ -105,6 -105,15 +105,15 @@@ // TODO: path detection, detection memory, etc. $oDB =& getDB(); + + $sVersionString = $oDB->getOne('select version()'); + preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches); + if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2]) + { + echo "ERROR: PostgreSQL version is not correct. Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n"; + exit; + } + passthru('createlang plpgsql '.$aDSNInfo['database']); $pgver = (float) CONST_Postgresql_Version; if ($pgver < 9.1) { @@@ -113,7 -122,16 +122,16 @@@ } else { pgsqlRunScript('CREATE EXTENSION hstore'); } + pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); + $sVersionString = $oDB->getOne('select postgis_full_version()'); + preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches); + if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2]) + { + echo "ERROR: PostGIS version is not correct. Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n"; + exit; + } + pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); @@@ -136,9 -154,8 +154,9 @@@ echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n"; fail("osm2pgsql not found in '$osm2pgsql'"); } + $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data'; $osm2pgsql .= ' -lsc -O gazetteer --hstore'; - $osm2pgsql .= ' -C '.$iCacheMemory; + $osm2pgsql .= ' -C 16000'; $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; passthruCheckReturn($osm2pgsql); @@@ -433,7 -450,7 +451,7 @@@ $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,"; $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,"; $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y "; - $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; + $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x"; if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; diff --combined website/reverse.php index 95889fc4,a0f5cea6..d43b0534 --- a/website/reverse.php +++ b/website/reverse.php @@@ -4,22 -4,6 +4,22 @@@ require_once(dirname(dirname(__FILE__)).'/lib/init-website.php'); require_once(CONST_BasePath.'/lib/log.php'); + if (preg_match(CONST_BlockedUserAgents, $_SERVER["HTTP_USER_AGENT"]) > 0) + { + $fLoadAvg = getLoadAverage(); + if ($fLoadAvg >= CONST_BlockReverseMaxLoad) { + header('HTTP/1.0 403 Forbidden'); + header('Content-type: text/html; charset=utf-8'); + echo "

App temporarily blocked

"; + echo "Your application has been temporarily blocked from the OpenStreetMap Nominatim "; + echo "geolocation service due to high server load."; + echo "\n\n"; + exit; + } + + } + + if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) { $fLoadAvg = getLoadAverage(); @@@ -115,7 -99,7 +115,7 @@@ $sSQL .= ' and (name is not null or housenumber is not null)'; $sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')'; $sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') '; - $sSQL .= ' OR ST_DWithin('.$sPointSQL.', ST_Centroid(geometry), '.$fSearchDiam.'))'; + $sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))'; $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1'; //var_dump($sSQL); $aPlace = $oDB->getRow($sSQL); @@@ -152,7 -136,7 +152,7 @@@ $sSQL .= " get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= " get_name_by_language(name, ARRAY['ref']) as ref,"; - $sSQL .= " st_y(st_centroid(geometry)) as lat, st_x(st_centroid(geometry)) as lon"; + $sSQL .= " st_y(centroid) as lat, st_x(centroid) as lon"; $sSQL .= " from placex where place_id = $iPlaceID "; $aPlace = $oDB->getRow($sSQL); diff --combined website/search.php index 24f21e12,5947fbc7..a92ac30a --- a/website/search.php +++ b/website/search.php @@@ -71,7 -71,6 +71,7 @@@ if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true; if (isset($aLangPrefOrder['name:ru'])) $bReverseInPlan = true; if (isset($aLangPrefOrder['name:ja'])) $bReverseInPlan = true; + if (isset($aLangPrefOrder['name:pl'])) $bReverseInPlan = true; $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]"; @@@ -325,7 -324,7 +325,7 @@@ { foreach($aSearchWords as $aSearchTerm) { - $aNewSearch = $aSearch; + $aNewSearch = $aSearch; if ($aSearchTerm['country_code']) { $aNewSearch['sCountryCode'] = strtolower($aSearchTerm['country_code']); @@@ -393,7 -392,7 +393,7 @@@ { // Check which tokens we have, get the ID numbers - $sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator'; + $sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator, search_name_count'; $sSQL .= ' from word where word_token in ('.join(',',array_map("getDBQuoted",$aTokens)).')'; $sSQL .= ' and search_name_count < '.CONST_Max_Word_Frequency; // $sSQL .= ' group by word_token, word, class, type, location, country_code'; @@@ -420,7 -419,7 +420,7 @@@ { $aValidTokens[$aToken['word_token']] = array($aToken); } - if ($aToken['word_token'][0]==' ' && !$aToken['class'] && !$aToken['country_code']) $aPossibleMainWordIDs[$aToken['word_id']] = 1; + if ($aToken['word_token'][0]==' ' && !$aToken['class'] && !$aToken['country_code']) $aPossibleMainWordIDs[$aToken['word_id']] = 1 + $aToken['search_name_count']; } if (CONST_Debug) var_Dump($aPhrases, $aValidTokens); @@@ -528,7 -527,7 +528,7 @@@ { $aSearch = $aCurrentSearch; $aSearch['iSearchRank']++; - if (($sPhraseType == '' || $sPhraseType == 'country') && $aSearchTerm['country_code'] !== null && $aSearchTerm['country_code'] != '0') + if (($sPhraseType == '' || $sPhraseType == 'country') && !empty($aSearchTerm['country_code']) && $aSearchTerm['country_code'] != '0') { if ($aSearch['sCountryCode'] === false) { @@@ -861,7 -860,7 +861,7 @@@ if (CONST_Debug) var_dump('
',$aSearch); if (CONST_Debug) _debugDumpGroupedSearches(array($iGroupedRank => array($aSearch)), $aValidTokens); $aPlaceIDs = array(); - + // First we need a position, either aName or fLat or both $aTerms = array(); $aOrder = array(); @@@ -869,7 -868,20 +869,20 @@@ // TODO: filter out the pointless search terms (2 letter name tokens and less) // they might be right - but they are just too darned expensive to run if (sizeof($aSearch['aName'])) $aTerms[] = "name_vector @> ARRAY[".join($aSearch['aName'],",")."]"; - if (sizeof($aSearch['aAddress']) && $aSearch['aName'] != $aSearch['aAddress']) $aTerms[] = "nameaddress_vector @> ARRAY[".join($aSearch['aAddress'],",")."]"; + if (sizeof($aSearch['aAddress']) && $aSearch['aName'] != $aSearch['aAddress']) + { + // For infrequent name terms disable index usage for address + if (CONST_Search_NameOnlySearchFrequencyThreshold && + sizeof($aSearch['aName']) == 1 && + $aPossibleMainWordIDs[$aSearch['aName'][reset($aSearch['aName'])]] < CONST_Search_NameOnlySearchFrequencyThreshold) + { + $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddress'],",")."]"; + } + else + { + $aTerms[] = "nameaddress_vector @> ARRAY[".join($aSearch['aAddress'],",")."]"; + } + } if ($aSearch['sCountryCode']) $aTerms[] = "country_code = '".pg_escape_string($aSearch['sCountryCode'])."'"; if ($aSearch['sHouseNumber']) $aTerms[] = "address_rank in (26,27)"; if ($aSearch['fLon'] && $aSearch['fLat']) @@@ -908,17 -920,12 +921,17 @@@ else $sSQL .= " limit ".$iLimit; - if (CONST_Debug) { var_dump($sSQL); } + if (CONST_Debug) var_dump($sSQL); + $iStartTime = time(); $aViewBoxPlaceIDs = $oDB->getAll($sSQL); if (PEAR::IsError($aViewBoxPlaceIDs)) { failInternalError("Could not get places for search terms.", $sSQL, $aViewBoxPlaceIDs); } + if (time() - $iStartTime > 60) { + file_put_contents(CONST_BasePath.'/log/long_queries.log', date('Y-m-d H:i:s', $iStartTime).' '.$sSQL."\n", FILE_APPEND); + } + //var_dump($aViewBoxPlaceIDs); // Did we have an viewbox matches? $aPlaceIDs = array(); @@@ -1047,13 -1054,13 +1060,13 @@@ if ($sNearPointSQL) $sOrderBySQL = "ST_Distance($sNearPointSQL, l.centroid)"; else if ($sPlaceIDs) $sOrderBySQL = "ST_Distance(l.centroid, f.geometry)"; else if ($sPlaceGeom) $sOrderBysSQL = "ST_Distance(st_centroid('".$sPlaceGeom."'), l.centroid)"; - + $sSQL = "select distinct l.place_id".($sOrderBySQL?','.$sOrderBySQL:'')." from place_classtype_".$aSearch['sClass']."_".$aSearch['sType']." as l"; if ($sCountryCodesSQL) $sSQL .= " join placex as lp using (place_id)"; if ($sPlaceIDs) { $sSQL .= ",placex as f where "; - $sSQL .= "f.place_id in ($sPlaceIDs) and ST_DWithin(l.centroid, st_centroid(f.geometry), $fRange) "; + $sSQL .= "f.place_id in ($sPlaceIDs) and ST_DWithin(l.centroid, f.centroid, $fRange) "; } if ($sPlaceGeom) { @@@ -1080,7 -1087,7 +1093,7 @@@ else $sOrderBySQL = "ST_Distance(l.geometry, f.geometry)"; $sSQL = "select distinct l.place_id".($sOrderBysSQL?','.$sOrderBysSQL:'')." from placex as l,placex as f where "; - $sSQL .= "f.place_id in ( $sPlaceIDs) and ST_DWithin(l.geometry, st_centroid(f.geometry), $fRange) "; + $sSQL .= "f.place_id in ( $sPlaceIDs) and ST_DWithin(l.geometry, f.centroid, $fRange) "; $sSQL .= "and l.class='".$aSearch['sClass']."' and l.type='".$aSearch['sType']."' "; if (sizeof($aExcludePlaceIDs)) { @@@ -1138,7 -1145,7 +1151,7 @@@ $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; - $sSQL .= "avg(ST_X(ST_Centroid(geometry))) as lon,avg(ST_Y(ST_Centroid(geometry))) as lat, "; + $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; // $sSQL .= $sOrderSQL." as porder, "; $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance "; $sSQL .= "from placex where place_id in ($sPlaceIDs) "; @@@ -1208,7 -1215,7 +1221,7 @@@ $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; - $sSQL .= "avg(ST_X(ST_Centroid(geometry))) as lon,avg(ST_Y(ST_Centroid(geometry))) as lat, "; + $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; // $sSQL .= $sOrderSQL." as porder, "; $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance "; $sSQL .= "from placex where place_id in ($sPlaceIDs) ";