From 274f38105249c24d355b1f46749c3841ffff6ff0 Mon Sep 17 00:00:00 2001 From: Brian Quinion Date: Sat, 8 Dec 2012 21:39:24 +0000 Subject: [PATCH] prune list of blocked items if too large, different costs for different page types --- lib/init.php | 1 + lib/leakybucket.php | 137 ++++++++++++++++++++++++++++++++++++++++++ settings/settings.php | 1 + utils/blocks.php | 14 ++++- website/details.php | 2 + website/reverse.php | 2 + website/search.php | 2 + 7 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 lib/leakybucket.php diff --git a/lib/init.php b/lib/init.php index 15e38a5b..67efdbfd 100644 --- a/lib/init.php +++ b/lib/init.php @@ -4,6 +4,7 @@ require_once(CONST_BasePath.'/settings/settings.php'); require_once(CONST_BasePath.'/lib/lib.php'); + require_once(CONST_BasePath.'/lib/leakybucket.php'); require_once(CONST_BasePath.'/lib/db.php'); if (get_magic_quotes_gpc()) diff --git a/lib/leakybucket.php b/lib/leakybucket.php new file mode 100644 index 00000000..778fe580 --- /dev/null +++ b/lib/leakybucket.php @@ -0,0 +1,137 @@ +addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort); + } + return $m; + } + + function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold) + { + $m = getBucketMemcache(); + if (!$m) return 0; + + $iMaxVal = 0; + $t = time(); + + foreach($asKey as $sKey) + { + $aCurrentBlock = $m->get($sKey); + if (!$aCurrentBlock) + { + $aCurrentBlock = array($iRequestCost, $t); + } + else + { + // add RequestCost + // remove leak * the time since the last request + $aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond; + $aCurrentBlock[1] = $t; + } + + if ($aCurrentBlock[0] <= 0) + { + $m->delete($sKey); + } + else + { + // If we have hit the threshold stop and record this to the block list + if ($aCurrentBlock[0] >= $iThreshold) + { + $aCurrentBlock[0] = $iThreshold; + + // Make up to 10 attempts to record this to memcache (with locking to prevent conflicts) + $i = 10; + for($i = 0; $i < 10; $i++) + { + $aBlockedList = $m->get('blockedList', null, $hCasToken); + if (!$aBlockedList) + { + $aBlockedList = array(); + $m->add('blockedList', $aBlockedList); + $aBlockedList = $m->get('blockedList', null, $hCasToken); + } + if (!isset($aBlockedList[$sKey])) + { + $aBlockedList[$sKey] = array(1, $t); + } + else + { + $aBlockedList[$sKey][0]++; + $aBlockedList[$sKey][1] = $t; + } + if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList) + { + uasort($aBlockedList, 'byValue1'); + $aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList); + } + $x = $m->cas($hCasToken, 'blockedList', $aBlockedList); + if ($x) break; + } + } + // Only keep in memcache until the time it would have expired (to avoid clutering memcache) + $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond); + } + + // Bucket result in the largest bucket we find + $iMaxVal = max($iMaxVal, $aCurrentBlock[0]); + } + + return $iMaxVal; + } + + function byValue1($a, $b) + { + if ($a[1] == $b[1]) + { + return 0; + } + return ($a[1] > $b[1]) ? -1 : 1; + } + + function byLastBlockTime($a, $b) + { + if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp']) + { + return 0; + } + return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1; + } + + function getBucketBlocks() + { + $m = getBucketMemcache(); + if (!$m) return null; + $t = time(); + $aBlockedList = $m->get('blockedList', null, $hCasToken); + if (!$aBlockedList) $aBlockedList = array(); + foreach($aBlockedList as $sKey => $aDetails) + { + $aCurrentBlock = $m->get($sKey); + if (!$aCurrentBlock) $aCurrentBlock = array(0, $t); + $iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate); + $aBlockedList[$sKey] = array( + 'totalBlocks' => $aDetails[0], + 'lastBlockTimestamp' => $aDetails[1], + 'currentBucketSize' => $iCurrentBucketSize, + 'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit, + ); + } + uasort($aBlockedList, 'byLastBlockTime'); + return $aBlockedList; + } + + function clearBucketBlocks() + { + $m = getBucketMemcache(); + if (!$m) return false; + $m->delete('blockedList'); + return true; + } diff --git a/settings/settings.php b/settings/settings.php index 6bf526ca..58463a51 100644 --- a/settings/settings.php +++ b/settings/settings.php @@ -17,6 +17,7 @@ // Connection buckets to rate limit people being nasty @define('CONST_ConnectionBucket_MemcacheServerAddress', false); @define('CONST_ConnectionBucket_MemcacheServerPort', 11211); + @define('CONST_ConnectionBucket_MaxBlockList', 100); @define('CONST_ConnectionBucket_LeakRate', 1); @define('CONST_ConnectionBucket_BlockLimit', 10); @define('CONST_ConnectionBucket_WaitLimit', 6); diff --git a/utils/blocks.php b/utils/blocks.php index d2db17f0..6dee2845 100755 --- a/utils/blocks.php +++ b/utils/blocks.php @@ -11,6 +11,7 @@ array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'), array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'), + array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'), ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true); @@ -28,11 +29,13 @@ $aBlocks = getBucketBlocks(); echo "\n"; - printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Req Blocked", "Last Block Time"); - printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-16s-|-%'-31s\n", "", "", "", "", "", ""); + printf(" %-40s | %12s | %7s | %13s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time"); + printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s\n", "", "", "", "", ""); foreach($aBlocks as $sKey => $aDetails) { - printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp'])); + printf(" %-40s | %12s | %7s | %13s | %31s\n", $sKey, $aDetails['totalBlocks'], + (int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N', + date("r", $aDetails['lastBlockTimestamp'])); } echo "\n"; } @@ -42,3 +45,8 @@ $m->set('sleepCounter', 0); clearBucketBlocks(); } + + if ($aResult['flush']) + { + $m->flush(); + } diff --git a/website/details.php b/website/details.php index 3d80ea5f..9cbbf28f 100755 --- a/website/details.php +++ b/website/details.php @@ -1,4 +1,6 @@