From: Sarah Hoffmann Date: Fri, 25 Jan 2013 18:56:31 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~645 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/5502c223756ed755edae9670497f1456e528d3f0?ds=inline;hp=-c Merge remote-tracking branch 'upstream/master' --- 5502c223756ed755edae9670497f1456e528d3f0 diff --combined settings/settings.php index 3de1e73b,fc3166b4..91595272 --- a/settings/settings.php +++ b/settings/settings.php @@@ -28,6 -28,7 +28,7 @@@ @define('CONST_ConnectionBucket_Cost_Reverse', 1); @define('CONST_ConnectionBucket_Cost_Search', 2); @define('CONST_ConnectionBucket_Cost_Details', 3); + @define('CONST_ConnectionBucket_Cost_Status', 1); // Override this function to add an adjustment factor to the cost // based on server load. e.g. getBlockingProcesses @@@ -41,14 -42,12 +42,14 @@@ // Website settings @define('CONST_NoAccessControl', true); - @define('CONST_ClosedForIndexing', false); - @define('CONST_ClosedForIndexingExceptionIPs', ''); @define('CONST_BlockedIPs', ''); + @define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks'); + @define('CONST_WhitelistedIPs', ''); + @define('CONST_BlockedUserAgents', ''); + @define('CONST_BlockReverseMaxLoad', 15); @define('CONST_BulkUserIPs', ''); - @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); + @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/'); @define('CONST_Tile_Default', 'Mapnik'); @define('CONST_Default_Language', 'xx'); diff --combined utils/setup.php index ba86e92c,e49a2d52..68e7a8b3 --- a/utils/setup.php +++ b/utils/setup.php @@@ -155,9 -155,8 +155,9 @@@ echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n"; fail("osm2pgsql not found in '$osm2pgsql'"); } + $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data'; $osm2pgsql .= ' -lsc -O gazetteer --hstore'; - $osm2pgsql .= ' -C '.$iCacheMemory; + $osm2pgsql .= ' -C 16000'; $osm2pgsql .= ' -P '.$aDSNInfo['port']; $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; passthruCheckReturn($osm2pgsql); @@@ -482,7 -481,7 +482,7 @@@ $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,"; $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,"; $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y "; - $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; + $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x"; if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; @@@ -620,6 -619,7 +620,7 @@@ @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php'); @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php'); @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php'); + @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php'); @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images'); @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js'); @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css'); diff --combined website/reverse.php index d43b0534,5eec5c74..859c7e3d --- a/website/reverse.php +++ b/website/reverse.php @@@ -4,22 -4,6 +4,22 @@@ require_once(dirname(dirname(__FILE__)).'/lib/init-website.php'); require_once(CONST_BasePath.'/lib/log.php'); + if (preg_match(CONST_BlockedUserAgents, $_SERVER["HTTP_USER_AGENT"]) > 0) + { + $fLoadAvg = getLoadAverage(); + if ($fLoadAvg >= CONST_BlockReverseMaxLoad) { + header('HTTP/1.0 403 Forbidden'); + header('Content-type: text/html; charset=utf-8'); + echo "

App temporarily blocked

"; + echo "Your application has been temporarily blocked from the OpenStreetMap Nominatim "; + echo "geolocation service due to high server load."; + echo "\n\n"; + exit; + } + + } + + if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) { $fLoadAvg = getLoadAverage(); @@@ -160,7 -144,7 +160,7 @@@ if ($bShowAddressDetails) { - $aAddress = getAddressDetails($oDB, $sLanguagePrefArraySQL, $iPlaceID, $aPlace['country_code']); + $aAddress = getAddressDetails($oDB, $sLanguagePrefArraySQL, $iPlaceID, $aPlace['calculated_country_code']); } $aClassType = getClassTypes(); $sAddressType = ''; diff --combined website/search.php index cfc1455f,a1860f25..77a3d135 --- a/website/search.php +++ b/website/search.php @@@ -71,7 -71,6 +71,7 @@@ if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true; if (isset($aLangPrefOrder['name:ru'])) $bReverseInPlan = true; if (isset($aLangPrefOrder['name:ja'])) $bReverseInPlan = true; + if (isset($aLangPrefOrder['name:pl'])) $bReverseInPlan = true; $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]"; @@@ -809,6 -808,10 +809,10 @@@ { $iQueryLoop++; + if (CONST_Debug) { echo "
Search Loop, group $iGroupLoop, loop $iQueryLoop"; } + if (CONST_Debug) _debugDumpGroupedSearches(array($iGroupedRank => array($aSearch)), $aValidTokens); + + // Must have a location term if (!sizeof($aSearch['aName']) && !sizeof($aSearch['aAddress']) && !$aSearch['fLon']) { @@@ -816,9 -819,10 +820,10 @@@ { if (4 >= $iMinAddressRank && 4 <= $iMaxAddressRank) { - $sSQL = "select place_id from placex where country_code='".$aSearch['sCountryCode']."' and rank_search = 4"; - if ($sCountryCodesSQL) $sSQL .= " and country_code in ($sCountryCodesSQL)"; + $sSQL = "select place_id from placex where calculated_country_code='".$aSearch['sCountryCode']."' and rank_search = 4"; + if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; $sSQL .= " order by st_area(geometry) desc limit 1"; + if (CONST_Debug) var_dump($sSQL); $aPlaceIDs = $oDB->getCol($sSQL); } } @@@ -826,15 -830,13 +831,13 @@@ { if (!$bBoundingBoxSearch && !$aSearch['fLon']) continue; if (!$aSearch['sClass']) continue; - if (CONST_Debug) var_dump('
',$aSearch); - if (CONST_Debug) _debugDumpGroupedSearches(array($iGroupedRank => array($aSearch)), $aValidTokens); $sSQL = "select count(*) from pg_tables where tablename = 'place_classtype_".$aSearch['sClass']."_".$aSearch['sType']."'"; if ($oDB->getOne($sSQL)) { $sSQL = "select place_id from place_classtype_".$aSearch['sClass']."_".$aSearch['sType']." ct"; if ($sCountryCodesSQL) $sSQL .= " join placex using (place_id)"; $sSQL .= " where st_contains($sViewboxSmallSQL, ct.centroid)"; - if ($sCountryCodesSQL) $sSQL .= " and country_code in ($sCountryCodesSQL)"; + if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; if (sizeof($aExcludePlaceIDs)) { $sSQL .= " and place_id not in (".join(',',$aExcludePlaceIDs).")"; @@@ -852,7 -854,7 +855,7 @@@ $sSQL = "select place_id from place_classtype_".$aSearch['sClass']."_".$aSearch['sType']." ct"; if ($sCountryCodesSQL) $sSQL .= " join placex using (place_id)"; $sSQL .= " where st_contains($sViewboxLargeSQL, ct.centroid)"; - if ($sCountryCodesSQL) $sSQL .= " and country_code in ($sCountryCodesSQL)"; + if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; if ($sViewboxCentreSQL) $sSQL .= " order by st_distance($sViewboxCentreSQL, ct.centroid) asc"; $sSQL .= " limit $iLimit"; if (CONST_Debug) var_dump($sSQL); @@@ -863,7 -865,7 +866,7 @@@ { $sSQL = "select place_id from placex where class='".$aSearch['sClass']."' and type='".$aSearch['sType']."'"; $sSQL .= " and st_contains($sViewboxSmallSQL, geometry) and linked_place_id is null"; - if ($sCountryCodesSQL) $sSQL .= " and country_code in ($sCountryCodesSQL)"; + if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; if ($sViewboxCentreSQL) $sSQL .= " order by st_distance($sViewboxCentreSQL, centroid) asc"; $sSQL .= " limit $iLimit"; if (CONST_Debug) var_dump($sSQL); @@@ -873,8 -875,6 +876,6 @@@ } else { - if (CONST_Debug) var_dump('
',$aSearch); - if (CONST_Debug) _debugDumpGroupedSearches(array($iGroupedRank => array($aSearch)), $aValidTokens); $aPlaceIDs = array(); // First we need a position, either aName or fLat or both @@@ -936,17 -936,12 +937,17 @@@ else $sSQL .= " limit ".$iLimit; - if (CONST_Debug) { var_dump($sSQL); } + if (CONST_Debug) var_dump($sSQL); + $iStartTime = time(); $aViewBoxPlaceIDs = $oDB->getAll($sSQL); if (PEAR::IsError($aViewBoxPlaceIDs)) { failInternalError("Could not get places for search terms.", $sSQL, $aViewBoxPlaceIDs); } + if (time() - $iStartTime > 60) { + file_put_contents(CONST_BasePath.'/log/long_queries.log', date('Y-m-d H:i:s', $iStartTime).' '.$sSQL."\n", FILE_APPEND); + } + //var_dump($aViewBoxPlaceIDs); // Did we have an viewbox matches? $aPlaceIDs = array(); @@@ -1023,7 -1018,7 +1024,7 @@@ // If they were searching for a named class (i.e. 'Kings Head pub') then we might have an extra match $sSQL = "select place_id from placex where place_id in ($sPlaceIDs) and class='".$aSearch['sClass']."' and type='".$aSearch['sType']."'"; $sSQL .= " and linked_place_id is null"; - if ($sCountryCodesSQL) $sSQL .= " and country_code in ($sCountryCodesSQL)"; + if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; $sSQL .= " order by rank_search asc limit $iLimit"; if (CONST_Debug) var_dump($sSQL); $aClassPlaceIDs = $oDB->getCol($sSQL); @@@ -1092,7 -1087,7 +1093,7 @@@ { $sSQL .= " and l.place_id not in (".join(',',$aExcludePlaceIDs).")"; } - if ($sCountryCodesSQL) $sSQL .= " and lp.country_code in ($sCountryCodesSQL)"; + if ($sCountryCodesSQL) $sSQL .= " and lp.calculated_country_code in ($sCountryCodesSQL)"; if ($sOrderBySQL) $sSQL .= "order by ".$sOrderBySQL." asc"; if ($iOffset) $sSQL .= " offset $iOffset"; $sSQL .= " limit $iLimit"; @@@ -1114,7 -1109,7 +1115,7 @@@ { $sSQL .= " and l.place_id not in (".join(',',$aExcludePlaceIDs).")"; } - if ($sCountryCodesSQL) $sSQL .= " and l.country_code in ($sCountryCodesSQL)"; + if ($sCountryCodesSQL) $sSQL .= " and l.calculated_country_code in ($sCountryCodesSQL)"; if ($sOrderBy) $sSQL .= "order by ".$OrderBysSQL." asc"; if ($iOffset) $sSQL .= " offset $iOffset"; $sSQL .= " limit $iLimit"; @@@ -1135,7 -1130,7 +1136,7 @@@ failInternalError("Could not get place IDs from tokens." ,$sSQL, $aPlaceIDs); } - if (CONST_Debug) var_Dump($aPlaceIDs); + if (CONST_Debug) { echo "
Place IDs: "; var_Dump($aPlaceIDs); } foreach($aPlaceIDs as $iPlaceID) { @@@ -1162,7 -1157,7 +1163,7 @@@ $sOrderSQL .= 'when min(place_id) = '.$iPlaceID.' then '.$iOrder.' '; } $sOrderSQL .= ' ELSE 10000000 END'; - $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,country_code,"; + $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,calculated_country_code as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; @@@ -1173,7 -1168,7 +1174,7 @@@ $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank "; if ($sAllowedTypesSQLList) $sSQL .= "and placex.class in $sAllowedTypesSQLList "; $sSQL .= "and linked_place_id is null "; - $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,country_code,importance"; + $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,calculated_country_code,importance"; if (!$bDeDupe) $sSQL .= ",place_id"; $sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) "; $sSQL .= ",get_name_by_language(name, $sLanguagePrefArraySQL) "; @@@ -1205,7 -1200,7 +1206,7 @@@ $sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) "; $sSQL .= "order by importance desc"; // $sSQL .= "order by rank_search,rank_address,porder asc"; - if (CONST_Debug) var_dump('
',$sSQL); + if (CONST_Debug) { echo "
"; var_dump($sSQL); } $aSearchResults = $oDB->getAll($sSQL); //var_dump($sSQL,$aSearchResults);exit; @@@ -1232,7 -1227,7 +1233,7 @@@ $sOrderSQL .= 'when min(place_id) = '.$iPlaceID.' then '.$iOrder.' '; } $sOrderSQL .= ' ELSE 10000000 END'; - $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,country_code,"; + $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,calculated_country_code as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; @@@ -1241,7 -1236,7 +1242,7 @@@ $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance "; $sSQL .= "from placex where place_id in ($sPlaceIDs) "; $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank "; - $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,country_code,importance"; + $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,calculated_country_code,importance"; if (!$bDeDupe) $sSQL .= ",place_id"; $sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) "; $sSQL .= ",get_name_by_language(name, $sLanguagePrefArraySQL) "; @@@ -1273,7 -1268,7 +1274,7 @@@ $sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) "; $sSQL .= "order by importance desc"; // $sSQL .= "order by rank_search,rank_address,porder asc"; - if (CONST_Debug) var_dump('
',$sSQL); + if (CONST_Debug) { echo "
", var_dump($sSQL); } $aSearchResults = $oDB->getAll($sSQL); //var_dump($sSQL,$aSearchResults);exit;