prune list of blocked items if too large, different costs for different page types

diff --git a/lib/init.php b/lib/init.php
index 15e38a5b17fcf7862e71bacda260db87e7b03d60..67efdbfdf6edb3c8894f7d8a487adfde7d193633 100644 (file)
--- a/lib/init.php
+++ b/lib/init.php
@@ -4,6 +4,7 @@
 
        require_once(CONST_BasePath.'/settings/settings.php');
        require_once(CONST_BasePath.'/lib/lib.php');
+       require_once(CONST_BasePath.'/lib/leakybucket.php');
        require_once(CONST_BasePath.'/lib/db.php');
 
        if (get_magic_quotes_gpc())

diff --git a/lib/leakybucket.php b/lib/leakybucket.php
new file mode 100644 (file)
index 0000000..778fe58
--- /dev/null
+++ b/lib/leakybucket.php
@@ -0,0 +1,137 @@
+<?php
+
+       function getBucketMemcache()
+       {
+               static $m;
+
+               if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
+               if (!isset($m))
+               {
+                       $m = new Memcached();
+                       $m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
+               }
+               return $m;
+       }
+
+       function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
+       {
+               $m = getBucketMemcache();
+               if (!$m) return 0;
+
+               $iMaxVal = 0;
+               $t = time();
+
+               foreach($asKey as $sKey)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock)
+                       {
+                               $aCurrentBlock = array($iRequestCost, $t);
+                       }
+                       else
+                       {
+                               // add RequestCost
+                               // remove leak * the time since the last request 
+                               $aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
+                               $aCurrentBlock[1] = $t;
+                       }
+
+                       if ($aCurrentBlock[0] <= 0)
+                       {
+                               $m->delete($sKey);
+                       }
+                       else
+                       {
+                               // If we have hit the threshold stop and record this to the block list
+                               if ($aCurrentBlock[0] >= $iThreshold)
+                               {
+                                       $aCurrentBlock[0] = $iThreshold;
+
+                                       // Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
+                                       $i = 10;
+                                       for($i = 0; $i < 10; $i++)
+                                       {
+                                               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               if (!$aBlockedList)
+                                               {
+                                                       $aBlockedList = array();
+                                                       $m->add('blockedList', $aBlockedList);
+                                                       $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               }
+                                               if (!isset($aBlockedList[$sKey]))
+                                               {
+                                                       $aBlockedList[$sKey] = array(1, $t);
+                                               }
+                                               else
+                                               {
+                                                       $aBlockedList[$sKey][0]++;
+                                                       $aBlockedList[$sKey][1] = $t;
+                                               }
+                                               if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
+                                               {
+                                                       uasort($aBlockedList, 'byValue1');
+                                                       $aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
+                                               }
+                                               $x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
+                                               if ($x) break;
+                                       }
+                               }
+                               // Only keep in memcache until the time it would have expired (to avoid clutering memcache)
+                               $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
+                       }
+
+                       // Bucket result in the largest bucket we find
+                       $iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
+               }
+
+               return $iMaxVal;
+        }
+
+       function byValue1($a, $b)
+       {
+               if ($a[1] == $b[1])
+               {
+                       return 0;
+               }
+               return ($a[1] > $b[1]) ? -1 : 1;
+       }
+
+       function byLastBlockTime($a, $b)
+       {
+               if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
+               {
+                       return 0;
+               }
+               return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
+       }
+
+       function getBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return null;
+               $t = time();
+               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+               if (!$aBlockedList) $aBlockedList = array();
+               foreach($aBlockedList as $sKey => $aDetails)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
+                       $iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
+                       $aBlockedList[$sKey] = array(
+                               'totalBlocks' => $aDetails[0],
+                               'lastBlockTimestamp' => $aDetails[1],
+                               'currentBucketSize' => $iCurrentBucketSize,
+                               'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
+                               );
+               }
+               uasort($aBlockedList, 'byLastBlockTime');
+               return $aBlockedList;
+       }
+
+       function clearBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return false;
+               $m->delete('blockedList');
+               return true;
+       }

diff --git a/settings/settings.php b/settings/settings.php
index 6bf526caaf8e05cadcec65c6a26eed0be0ffbdbe..58463a516c2a28cbe147a92e7f92e56a186f1abd 100644 (file)
--- a/settings/settings.php
+++ b/settings/settings.php
@@ -17,6 +17,7 @@
        // Connection buckets to rate limit people being nasty
        @define('CONST_ConnectionBucket_MemcacheServerAddress', false);
        @define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
+       @define('CONST_ConnectionBucket_MaxBlockList', 100);
        @define('CONST_ConnectionBucket_LeakRate', 1);
        @define('CONST_ConnectionBucket_BlockLimit', 10);
        @define('CONST_ConnectionBucket_WaitLimit', 6);

diff --git a/utils/blocks.php b/utils/blocks.php
index d2db17f0e9478b60e9dbc8e9f9faf8cb3a8e6c1b..6dee2845b06bfeb315da6af32b08c07ae5f7f4e1 100755 (executable)
--- a/utils/blocks.php
+++ b/utils/blocks.php
@@ -11,6 +11,7 @@
                array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
                array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
                array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
+               array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),
        );
        getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
@@ -28,11 +29,13 @@
 
                $aBlocks = getBucketBlocks();
                echo "\n";
-               printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Req Blocked", "Last Block Time");
-               printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-16s-|-%'-31s\n", "", "", "", "", "", "");
+               printf(" %-40s | %12s | %7s | %13s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time");
+               printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s\n", "", "", "", "", "");
                foreach($aBlocks as $sKey => $aDetails)
                {
-                       printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp']));
+                       printf(" %-40s | %12s | %7s | %13s | %31s\n", $sKey, $aDetails['totalBlocks'], 
+                               (int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N', 
+                               date("r", $aDetails['lastBlockTimestamp']));
                }
                echo "\n";
        }
@@ -42,3 +45,8 @@
                $m->set('sleepCounter', 0);
                clearBucketBlocks();
        }
+
+       if ($aResult['flush'])
+       {
+               $m->flush();
+       }

diff --git a/website/details.php b/website/details.php
index 3d80ea5f7b96215f756790589c34bd65ab5c90b6..9cbbf28f87b94182c99c3eedc6b327f495c0affc 100755 (executable)
--- a/website/details.php
+++ b/website/details.php
@@ -1,4 +1,6 @@
 <?php
+       @define('CONST_ConnectionBucket_PageType', 'Details');
+
         require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
         require_once(CONST_BasePath.'/lib/log.php');
 

diff --git a/website/reverse.php b/website/reverse.php
index c155fa573030fa54cb5a71629727b808dc2be972..d5a36998efd4143a7513dc6f5f8a30797f9e4aab 100755 (executable)
--- a/website/reverse.php
+++ b/website/reverse.php
@@ -1,4 +1,6 @@
 <?php
+       @define('CONST_ConnectionBucket_PageType', 'Reverse');
+
        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');
 

diff --git a/website/search.php b/website/search.php
index 08c2eee095049c3ff6090c3de03bb42a265e6d8c..e69ea78d4d3ed126af55ef656e6bdf13752c45d4 100755 (executable)
--- a/website/search.php
+++ b/website/search.php
@@ -1,4 +1,6 @@
 <?php
+       @define('CONST_ConnectionBucket_PageType', 'Search');
+
        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');