From: Sarah Hoffmann Date: Mon, 18 Jul 2016 12:41:09 +0000 (+0200) Subject: remove IP blocking and rate-limiting code X-Git-Tag: v3.0.0~153 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/4f75f4cb6e468a8de282a81615e7c720e154c8b4?ds=inline;hp=-c remove IP blocking and rate-limiting code Closes #472. --- 4f75f4cb6e468a8de282a81615e7c720e154c8b4 diff --git a/lib/init-website.php b/lib/init-website.php index 04bc518c..61a41731 100644 --- a/lib/init-website.php +++ b/lib/init-website.php @@ -13,47 +13,5 @@ } if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit; - $aBucketKeys = array(); - - if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST))); - if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"]; - if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"]; - - $fBucketVal = doBucket($aBucketKeys, - (defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(), - CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit); - - if ($fBucketVal > CONST_ConnectionBucket_WaitLimit && $fBucketVal < CONST_ConnectionBucket_BlockLimit) - { - $m = getBucketMemcache(); - $iCurrentSleeping = $m->increment('sleepCounter'); - if (false === $iCurrentSleeping) - { - $m->add('sleepCounter', 0); - $iCurrentSleeping = $m->increment('sleepCounter'); - } - if ($iCurrentSleeping >= CONST_ConnectionBucket_MaxSleeping || isBucketSleeping($aBucketKeys)) - { - // Too many threads sleeping already. This becomes a hard block. - $fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_BlockLimit, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit); - } - else - { - setBucketSleeping($aBucketKeys, true); - sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate); - $fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit); - setBucketSleeping($aBucketKeys, false); - } - $m->decrement('sleepCounter'); - } - - if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit) - { - header("HTTP/1.0 429 Too Many Requests"); - echo "Your IP has been blocked. \n"; - echo CONST_BlockMessage; - exit; - } - header('Content-type: text/html; charset=utf-8'); diff --git a/lib/init.php b/lib/init.php index 26952d05..1d47c98f 100644 --- a/lib/init.php +++ b/lib/init.php @@ -1,7 +1,6 @@ addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort); - } - return $m; - } - - function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold) - { - $m = getBucketMemcache(); - if (!$m) return 0; - - $iMaxVal = 0; - $t = time(); - - foreach($asKey as $sKey) - { - $aCurrentBlock = $m->get($sKey); - if (!$aCurrentBlock) - { - $aCurrentBlock = array($iRequestCost, $t, false); - } - else - { - // add RequestCost - // remove leak * the time since the last request - $aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond; - $aCurrentBlock[1] = $t; - } - - if ($aCurrentBlock[0] <= 0) - { - $m->delete($sKey); - } - else - { - // If we have hit the threshold stop and record this to the block list - if ($aCurrentBlock[0] >= $iThreshold) - { - $aCurrentBlock[0] = $iThreshold; - - // Make up to 10 attempts to record this to memcache (with locking to prevent conflicts) - $i = 10; - for($i = 0; $i < 10; $i++) - { - $aBlockedList = $m->get('blockedList', null, $hCasToken); - if (!$aBlockedList) - { - $aBlockedList = array(); - $m->add('blockedList', $aBlockedList); - $aBlockedList = $m->get('blockedList', null, $hCasToken); - } - if (!isset($aBlockedList[$sKey])) - { - $aBlockedList[$sKey] = array(1, $t); - } - else - { - $aBlockedList[$sKey][0]++; - $aBlockedList[$sKey][1] = $t; - } - if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList) - { - uasort($aBlockedList, 'byValue1'); - $aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList); - } - $x = $m->cas($hCasToken, 'blockedList', $aBlockedList); - if ($x) break; - } - } - // Only keep in memcache until the time it would have expired (to avoid clutering memcache) - $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond); - } - - // Bucket result in the largest bucket we find - $iMaxVal = max($iMaxVal, $aCurrentBlock[0]); - } - - return $iMaxVal; - } - - function isBucketSleeping($asKey) - { - $m = getBucketMemcache(); - if (!$m) return false; - - foreach($asKey as $sKey) - { - $aCurrentBlock = $m->get($sKey); - if ($aCurrentBlock[2]) return true; - } - return false; - } - - function setBucketSleeping($asKey, $bVal) - { - $m = getBucketMemcache(); - if (!$m) return false; - - $iMaxVal = 0; - $t = time(); - - foreach($asKey as $sKey) - { - $aCurrentBlock = $m->get($sKey); - $aCurrentBlock[2] = $bVal; - $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/CONST_ConnectionBucket_LeakRate); - } - return true; - } - - function byValue1($a, $b) - { - if ($a[1] == $b[1]) - { - return 0; - } - return ($a[1] > $b[1]) ? -1 : 1; - } - - function byLastBlockTime($a, $b) - { - if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp']) - { - return 0; - } - return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1; - } - - function getBucketBlocks() - { - $m = getBucketMemcache(); - if (!$m) return null; - $t = time(); - $aBlockedList = $m->get('blockedList', null, $hCasToken); - if (!$aBlockedList) $aBlockedList = array(); - foreach($aBlockedList as $sKey => $aDetails) - { - $aCurrentBlock = $m->get($sKey); - if (!$aCurrentBlock) $aCurrentBlock = array(0, $t); - $iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate); - $aBlockedList[$sKey] = array( - 'totalBlocks' => $aDetails[0], - 'lastBlockTimestamp' => $aDetails[1], - 'isSleeping' => (isset($aCurrentBlock[2])?$aCurrentBlock[2]:false), - 'currentBucketSize' => $iCurrentBucketSize, - 'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit, - ); - } - uasort($aBlockedList, 'byLastBlockTime'); - return $aBlockedList; - } - - function clearBucketBlocks() - { - $m = getBucketMemcache(); - if (!$m) return false; - $m->delete('blockedList'); - return true; - } diff --git a/lib/lib.php b/lib/lib.php index ec5b9b1f..1f772331 100644 --- a/lib/lib.php +++ b/lib/lib.php @@ -9,14 +9,6 @@ } - function getLoadAverage() - { - $sLoadAverage = file_get_contents('/proc/loadavg'); - $aLoadAverage = explode(' ',$sLoadAverage); - return (float)$aLoadAverage[0]; - } - - function getProcessorCount() { $sCPU = file_get_contents('/proc/cpuinfo'); diff --git a/settings/settings.php b/settings/settings.php index d44bb967..cdb02924 100644 --- a/settings/settings.php +++ b/settings/settings.php @@ -62,34 +62,8 @@ @define('CONST_Replication_Update_Interval', '60'); // How often upstream publishes diffs @define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet - // Connection buckets to rate limit people being nasty - @define('CONST_ConnectionBucket_MemcacheServerAddress', false); - @define('CONST_ConnectionBucket_MemcacheServerPort', 11211); - @define('CONST_ConnectionBucket_MaxBlockList', 100); - @define('CONST_ConnectionBucket_LeakRate', 1); - @define('CONST_ConnectionBucket_BlockLimit', 10); - @define('CONST_ConnectionBucket_WaitLimit', 6); - @define('CONST_ConnectionBucket_MaxSleeping', 10); - @define('CONST_ConnectionBucket_Cost_Reverse', 1); - @define('CONST_ConnectionBucket_Cost_Search', 2); - @define('CONST_ConnectionBucket_Cost_Details', 3); - @define('CONST_ConnectionBucket_Cost_Status', 1); - - // Override this function to add an adjustment factor to the cost - // based on server load. e.g. getBlockingProcesses - if (!function_exists('user_busy_cost')) - { - function user_busy_cost() - { - return 0; - } - } - // Website settings @define('CONST_NoAccessControl', true); - @define('CONST_BlockedIPs', ''); - @define('CONST_BulkUserIPs', ''); - @define('CONST_BlockMessage', ''); // additional info to show for blocked IPs @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); // Language to assume when none is supplied with the query. diff --git a/website/details.php b/website/details.php index 495db2b4..10cd07e1 100755 --- a/website/details.php +++ b/website/details.php @@ -7,14 +7,7 @@ require_once(CONST_BasePath.'/lib/output.php'); $sOutputFormat = 'html'; - /* - $fLoadAvg = getLoadAverage(); - if ($fLoadAvg > 3) - { - echo "Page temporarily blocked due to high server load\n"; - exit; - } - */ + ini_set('memory_limit', '200M'); $oDB =& getDB(); diff --git a/website/lookup.php b/website/lookup.php index 9d0cea4c..2e2a1c2f 100755 --- a/website/lookup.php +++ b/website/lookup.php @@ -7,18 +7,6 @@ require_once(CONST_BasePath.'/lib/PlaceLookup.php'); require_once(CONST_BasePath.'/lib/output.php'); - if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) - { - $fLoadAvg = getLoadAverage(); - if ($fLoadAvg > 2) sleep(60); - if ($fLoadAvg > 4) sleep(120); - if ($fLoadAvg > 6) - { - userError("Bulk User: Temporary block due to high server load"); - exit; - } - } - $oDB =& getDB(); ini_set('memory_limit', '200M'); diff --git a/website/reverse.php b/website/reverse.php index 7f5eca78..0e0f2ff8 100755 --- a/website/reverse.php +++ b/website/reverse.php @@ -8,19 +8,6 @@ require_once(CONST_BasePath.'/lib/ReverseGeocode.php'); require_once(CONST_BasePath.'/lib/output.php'); - if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) - { - $fLoadAvg = getLoadAverage(); - if ($fLoadAvg > 2) sleep(60); - if ($fLoadAvg > 4) sleep(120); - if ($fLoadAvg > 6) - { - echo "Bulk User: Temporary block due to high server load\n"; - exit; - } - } - - $bAsGeoJSON = getParamBool('polygon_geojson'); $bAsKML = getParamBool('polygon_kml'); $bAsSVG = getParamBool('polygon_svg');