From e03ae6d7ebdf56d084b00091382f5bcf0cb7e486 Mon Sep 17 00:00:00 2001 From: Brian Quinion Date: Sat, 8 Dec 2012 17:38:10 +0000 Subject: [PATCH] rate limiting using memcache --- lib/init-website.php | 22 ++++++++- lib/lib.php | 109 +++++++++++++++++++++++++++++++++++++++++- settings/settings.php | 21 +++++++- utils/blocks.php | 40 ++++++++++++++++ 4 files changed, 188 insertions(+), 4 deletions(-) create mode 100755 utils/blocks.php diff --git a/lib/init-website.php b/lib/init-website.php index d21ca7af..896a7b24 100644 --- a/lib/init-website.php +++ b/lib/init-website.php @@ -1,5 +1,8 @@ x"; +//exit; +//phpinfo(); +//exit; require_once('init.php'); if (CONST_ClosedForIndexing && strpos(CONST_ClosedForIndexingExceptionIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false) @@ -8,7 +11,16 @@ exit; } - if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) + $aBucketKeys = array(); + if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST))); + if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"]; + if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"]; + + $fBucketVal = doBucket($aBucketKeys, + (defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(), + CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit); + + if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit) { echo "Your IP has been blocked. \n"; echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n"; @@ -16,4 +28,10 @@ exit; } + if ($fBucketVal > CONST_ConnectionBucket_WaitLimit) + { + sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate); + } +var_dump($fBucketVal); +exit; header('Content-type: text/html; charset=utf-8'); diff --git a/lib/lib.php b/lib/lib.php index f34b09b7..20f5af17 100644 --- a/lib/lib.php +++ b/lib/lib.php @@ -60,7 +60,7 @@ { $sLoadAverage = file_get_contents('/proc/loadavg'); $aLoadAverage = explode(' ',$sLoadAverage); - return (int)$aLoadAverage[0]; + return (float)$aLoadAverage[0]; } function getProcessorCount() @@ -917,3 +917,110 @@ return true; } + function getBucketMemcache() + { + if (!CONST_ConnectionBucket_MemcacheServerAddress) return null; + $m = new Memcached(); + $m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort); + return $m; + } + + function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold) + { + $m = getBucketMemcache(); + if (!$m) return 0; + + $iMaxVal = 0; + $t = time(); + + foreach($asKey as $sKey) + { + $aCurrentBlock = $m->get($sKey); + if (!$aCurrentBlock) + { + $aCurrentBlock = array($iRequestCost, $t); + } + else + { + // add RequestCost + // remove leak * the time since the last request + $aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond; + $aCurrentBlock[1] = $t; + } + + if ($aCurrentBlock[0] <= 0) + { + $m->delete($sKey); + } + else + { + // If we have hit the threshold stop and record this to the block list + if ($aCurrentBlock[0] >= $iThreshold) + { + $aCurrentBlock[0] = $iThreshold; + + // Make up to 10 attempts to record this to memcache (with locking to prevent conflicts) + $i = 10; + for($i = 0; $i < 10; $i++) + { + $aBlockedList = $m->get('blockedList', null, $hCasToken); + if (!$aBlockedList) + { + $aBlockedList = array(); + $m->add('blockedList', $aBlockedList); + $aBlockedList = $m->get('blockedList', null, $hCasToken); + } + if (!isset($aBlockedList[$sKey])) + { + $aBlockedList[$sKey] = array(1, $t); + } + else + { + $aBlockedList[$sKey][0]++; + $aBlockedList[$sKey][1] = $t; + } + $x = $m->cas($hCasToken, 'blockedList', $aBlockedList); + if ($x) break; + } + } + // Only keep in memcache until the time it would have expired (to avoid clutering memcache) + $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond); + } + + // Bucket result in the largest bucket we find + $iMaxVal = max($iMaxVal, $aCurrentBlock[0]); + } + + return $iMaxVal; + } + + function getBucketBlocks() + { + $m = getBucketMemcache(); + if (!$m) return null; + $t = time(); + $aBlockedList = $m->get('blockedList', null, $hCasToken); + if (!$aBlockedList) $aBlockedList = array(); + foreach($aBlockedList as $sKey => $aDetails) + { + $aCurrentBlock = $m->get($sKey); + if (!$aCurrentBlock) $aCurrentBlock = array(0, $t); + $iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate); + $aBlockedList[$sKey] = array( + 'totalBlocks' => $aDetails[0], + 'lastBlockTimestamp' => $aDetails[1], + 'currentBucketSize' => $iCurrentBucketSize, + 'lastRequestBlocked' => $aCurrentBlock[0] >= CONST_ConnectionBucket_BlockLimit, + 'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit, + ); + } + return $aBlockedList; + } + + function clearBucketBlocks() + { + $m = getBucketMemcache(); + if (!$m) return false; + $m->delete('blockedList'); + return true; + } diff --git a/settings/settings.php b/settings/settings.php index 0a214ce9..65604035 100644 --- a/settings/settings.php +++ b/settings/settings.php @@ -14,6 +14,24 @@ @define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql'); @define('CONST_Osmosis_Binary', '/usr/bin/osmosis'); + // Connection buckets to rate limit people being nasty + @define('CONST_ConnectionBucket_MemcacheServerAddress', false); + @define('CONST_ConnectionBucket_MemcacheServerPort', 11211); + @define('CONST_ConnectionBucket_LeakRate', 1); + @define('CONST_ConnectionBucket_BlockLimit', 10); + @define('CONST_ConnectionBucket_WaitLimit', 6); + @define('CONST_ConnectionBucket_Cost_Reverse', 1); + @define('CONST_ConnectionBucket_Cost_Search', 2); + @define('CONST_ConnectionBucket_Cost_Details', 3); + + if (!function_exists('user_busy_cost')) + { + function user_busy_cost() + { + return 0; + } + } + // Website settings @define('CONST_ClosedForIndexing', false); @define('CONST_ClosedForIndexingExceptionIPs', ''); @@ -30,10 +48,11 @@ @define('CONST_Search_AreaPolygons_Enabled', true); @define('CONST_Search_AreaPolygons', true); - @define('CONST_Search_TryDroppedAddressTerms', false); @define('CONST_Suggestions_Enabled', false); + @define('CONST_Search_TryDroppedAddressTerms', false); + // Set to zero to disable polygon output @define('CONST_PolygonOutput_MaximumTypes', 1); diff --git a/utils/blocks.php b/utils/blocks.php new file mode 100755 index 00000000..4d323115 --- /dev/null +++ b/utils/blocks.php @@ -0,0 +1,40 @@ +#!/usr/bin/php -Cq + $aDetails) + { + printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp'])); + } + echo "\n"; + } + + if ($aResult['delete']) + { + clearBucketBlocks(); + } -- 2.39.5