From: Sarah Hoffmann Date: Wed, 12 Dec 2012 17:55:03 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~663 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/7c56f24c1f83e9d19c9df723bb86335598add902?hp=-c Merge remote-tracking branch 'upstream/master' Conflicts: lib/init-website.php --- 7c56f24c1f83e9d19c9df723bb86335598add902 diff --combined lib/init-website.php index 6db2ac37,8603e309..ef8237fb --- a/lib/init-website.php +++ b/lib/init-website.php @@@ -1,34 -1,53 +1,51 @@@

Access blocked

"; - echo "Your IP has been blocked for overusing OpenStreetMap's volunteer-run servers.
\n"; - echo 'Please consult the Nominatim usage policy for more information.'; - echo "\n\n"; - exit; - } + $aBucketKeys = array(); - $sTempBlockedIP = file_get_contents(CONST_IPBanFile); - if (preg_match('/\b'.$_SERVER["REMOTE_ADDR"].'\b/', $sTempBlockedIP)) - { - header('HTTP/1.0 503 Service Temporarily Unavailable'); - header('Content-type: text/html; charset=utf-8'); - echo "

Access blocked

"; - echo "Your IP has been blocked temporarily for overusing OpenStreetMap's volunteer-run servers. This ban will be lifted automatically in a while. To avoid further blocks, please read the
\n"; - echo 'Nominatim usage policy carefully before you continue to use this service.'; - echo "\n\n"; - exit; - } + if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST))); + if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"]; + if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"]; - } + $fBucketVal = doBucket($aBucketKeys, + (defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(), + CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit); + + if ($fBucketVal > CONST_ConnectionBucket_WaitLimit && $fBucketVal < CONST_ConnectionBucket_BlockLimit) + { + $m = getBucketMemcache(); + $iCurrentSleeping = $m->increment('sleepCounter'); + if (false === $iCurrentSleeping) + { + $m->add('sleepCounter', 0); + $iCurrentSleeping = $m->increment('sleepCounter'); + } + if ($iCurrentSleeping >= CONST_ConnectionBucket_MaxSleeping || isBucketSleeping($aBucketKeys)) + { + // Too many threads sleeping already. This becomes a hard block. + $fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_BlockLimit, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit); + } + else + { + setBucketSleeping($aBucketKeys, true); + sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate); + $fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit); + setBucketSleeping($aBucketKeys, false); + } + $m->decrement('sleepCounter'); + } + + if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit) + { + echo "Your IP has been blocked. \n"; + echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n"; + echo "Information on the Nominatim usage policy can be found here: http://wiki.openstreetmap.org/wiki/Nominatim#Usage_Policy \n"; + exit; + } - - header('Content-type: text/html; charset=utf-8'); - diff --combined settings/settings.php index 8ebd81ee,58463a51..d82a3337 --- a/settings/settings.php +++ b/settings/settings.php @@@ -14,28 -14,49 +14,51 @@@ @define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql'); @define('CONST_Osmosis_Binary', '/usr/bin/osmosis'); + // Connection buckets to rate limit people being nasty + @define('CONST_ConnectionBucket_MemcacheServerAddress', false); + @define('CONST_ConnectionBucket_MemcacheServerPort', 11211); + @define('CONST_ConnectionBucket_MaxBlockList', 100); + @define('CONST_ConnectionBucket_LeakRate', 1); + @define('CONST_ConnectionBucket_BlockLimit', 10); + @define('CONST_ConnectionBucket_WaitLimit', 6); + @define('CONST_ConnectionBucket_MaxSleeping', 10); + @define('CONST_ConnectionBucket_Cost_Reverse', 1); + @define('CONST_ConnectionBucket_Cost_Search', 2); + @define('CONST_ConnectionBucket_Cost_Details', 3); + + // Override this function to add an adjustment factor to the cost + // based on server load. e.g. getBlockingProcesses + if (!function_exists('user_busy_cost')) + { + function user_busy_cost() + { + return 0; + } + } + // Website settings - @define('CONST_ClosedForIndexing', false); - @define('CONST_ClosedForIndexingExceptionIPs', ''); @define('CONST_BlockedIPs', ''); + @define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks'); + @define('CONST_WhitelistedIPs', ''); + @define('CONST_BlockedUserAgents', ''); + @define('CONST_BlockReverseMaxLoad', 15); @define('CONST_BulkUserIPs', ''); - @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); + @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/'); @define('CONST_Tile_Default', 'Mapnik'); - @define('CONST_Default_Language', 'xx'); + @define('CONST_Default_Language', 'en'); @define('CONST_Default_Lat', 20.0); @define('CONST_Default_Lon', 0.0); @define('CONST_Default_Zoom', 2); @define('CONST_Search_AreaPolygons_Enabled', true); @define('CONST_Search_AreaPolygons', true); - @define('CONST_Search_TryDroppedAddressTerms', false); @define('CONST_Suggestions_Enabled', false); + @define('CONST_Search_TryDroppedAddressTerms', false); + // Set to zero to disable polygon output @define('CONST_PolygonOutput_MaximumTypes', 1); diff --combined website/reverse.php index 810be3c4,d5a36998..2c62948f --- a/website/reverse.php +++ b/website/reverse.php @@@ -1,23 -1,9 +1,25 @@@ 0) + { + $fLoadAvg = getLoadAverage(); + if ($fLoadAvg >= CONST_BlockReverseMaxLoad) { + header('HTTP/1.0 403 Forbidden'); + header('Content-type: text/html; charset=utf-8'); + echo "

App temporarily blocked

"; + echo "Your application has been temporarily blocked from the OpenStreetMap Nominatim "; + echo "geolocation service due to high server load."; + echo "\n\n"; + exit; + } + + } + + if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) { $fLoadAvg = getLoadAverage(); diff --combined website/search.php index b7a59613,5ba2571b..24f21e12 --- a/website/search.php +++ b/website/search.php @@@ -1,8 -1,11 +1,11 @@@ 50) $iFinalLimit = 50; - $iLimit = $iFinalLimit + min($iFinalLimit, 10); + $iLimit = $iFinalLimit + min($iFinalLimit, 10); $iMinAddressRank = 0; $iMaxAddressRank = 30; + $sAllowedTypesSQLList = false; // Format for output if (isset($_GET['format']) && ($_GET['format'] == 'html' || $_GET['format'] == 'xml' || $_GET['format'] == 'json' || $_GET['format'] == 'jsonv2')) @@@ -67,7 -71,6 +71,7 @@@ if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true; if (isset($aLangPrefOrder['name:ru'])) $bReverseInPlan = true; if (isset($aLangPrefOrder['name:ja'])) $bReverseInPlan = true; + if (isset($aLangPrefOrder['name:pl'])) $bReverseInPlan = true; $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]"; @@@ -153,12 -156,14 +157,14 @@@ array('postalcode', 16, 25), ); $aStructuredQuery = array(); + $sAllowedTypesSQLList = ''; foreach($aStructuredOptions as $aStructuredOption) { loadStructuredAddressElement($aStructuredQuery, $iMinAddressRank, $iMaxAddressRank, $_GET, $aStructuredOption[0], $aStructuredOption[1], $aStructuredOption[2]); } if (sizeof($aStructuredQuery) > 0) { $sQuery = join(', ', $aStructuredQuery); + $sAllowedTypesSQLList = '(\'place\',\'boundary\')'; } if ($sQuery) @@@ -883,7 -888,7 +889,7 @@@ if ($bBoundingBoxSearch) $aTerms[] = "centroid && $sViewboxSmallSQL"; if ($sNearPointSQL) $aOrder[] = "ST_Distance($sNearPointSQL, centroid) asc"; - $sImportanceSQL = 'case when importance = 0 OR importance IS NULL then 0.92-(search_rank::float/33) else importance end'; + $sImportanceSQL = 'case when importance = 0 OR importance IS NULL then 0.75-(search_rank::float/40) else importance end'; if ($sViewboxSmallSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxSmallSQL, centroid) THEN 1 ELSE 0.5 END"; if ($sViewboxLargeSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxLargeSQL, centroid) THEN 1 ELSE 0.5 END"; @@@ -902,17 -907,12 +908,17 @@@ else $sSQL .= " limit ".$iLimit; - if (CONST_Debug) { var_dump($sSQL); } + if (CONST_Debug) var_dump($sSQL); + $iStartTime = time(); $aViewBoxPlaceIDs = $oDB->getAll($sSQL); if (PEAR::IsError($aViewBoxPlaceIDs)) { failInternalError("Could not get places for search terms.", $sSQL, $aViewBoxPlaceIDs); } + if (time() - $iStartTime > 60) { + file_put_contents(CONST_BasePath.'/log/long_queries.log', date('Y-m-d H:i:s', $iStartTime).' '.$sSQL."\n", FILE_APPEND); + } + //var_dump($aViewBoxPlaceIDs); // Did we have an viewbox matches? $aPlaceIDs = array(); @@@ -1134,9 -1134,10 +1140,10 @@@ $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; $sSQL .= "avg(ST_X(ST_Centroid(geometry))) as lon,avg(ST_Y(ST_Centroid(geometry))) as lat, "; // $sSQL .= $sOrderSQL." as porder, "; - $sSQL .= "coalesce(importance,0.9-(rank_search::float/30)) as importance "; + $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance "; $sSQL .= "from placex where place_id in ($sPlaceIDs) "; $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank "; + if ($sAllowedTypesSQLList) $sSQL .= "and placex.class in $sAllowedTypesSQLList "; $sSQL .= "and linked_place_id is null "; $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,country_code,importance"; if (!$bDeDupe) $sSQL .= ",place_id"; @@@ -1203,7 -1204,7 +1210,7 @@@ $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; $sSQL .= "avg(ST_X(ST_Centroid(geometry))) as lon,avg(ST_Y(ST_Centroid(geometry))) as lat, "; // $sSQL .= $sOrderSQL." as porder, "; - $sSQL .= "coalesce(importance,0.9-(rank_search::float/30)) as importance "; + $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance "; $sSQL .= "from placex where place_id in ($sPlaceIDs) "; $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank "; $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,country_code,importance"; @@@ -1433,8 -1434,6 +1440,6 @@@ } uasort($aSearchResults, 'byImportance'); - //var_dump($aSearchResults);exit; - $aOSMIDDone = array(); $aClassTypeNameDone = array(); $aToFilter = $aSearchResults; @@@ -1476,7 -1475,7 +1481,7 @@@ logEnd($oDB, $hLog, sizeof($aToFilter)); } $sMoreURL = CONST_Website_BaseURL.'search?format='.urlencode($sOutputFormat).'&exclude_place_ids='.join(',',$aExcludePlaceIDs); - $sMoreURL .= '&accept-language='.$_SERVER["HTTP_ACCEPT_LANGUAGE"]; + if (isset($_SERVER["HTTP_ACCEPT_LANGUAGE"])) $sMoreURL .= '&accept-language='.$_SERVER["HTTP_ACCEPT_LANGUAGE"]; if ($bShowPolygons) $sMoreURL .= '&polygon=1'; if ($bShowAddressDetails) $sMoreURL .= '&addressdetails=1'; if (isset($_GET['viewbox']) && $_GET['viewbox']) $sMoreURL .= '&viewbox='.urlencode($_GET['viewbox']);