prune list of blocked items if too large, different costs for different page types

diff --git a/lib/init.php b/lib/init.php
index 15e38a5b17fcf7862e71bacda260db87e7b03d60..67efdbfdf6edb3c8894f7d8a487adfde7d193633 100644 (file)
--- a/lib/init.php
+++ b/lib/init.php
@@ -4,6 +4,7 @@
 
        require_once(CONST_BasePath.'/settings/settings.php');
        require_once(CONST_BasePath.'/lib/lib.php');
 
        require_once(CONST_BasePath.'/settings/settings.php');
        require_once(CONST_BasePath.'/lib/lib.php');

+       require_once(CONST_BasePath.'/lib/leakybucket.php');

        require_once(CONST_BasePath.'/lib/db.php');
 
        if (get_magic_quotes_gpc())
        require_once(CONST_BasePath.'/lib/db.php');
 
        if (get_magic_quotes_gpc())

diff --git a/lib/leakybucket.php b/lib/leakybucket.php
new file mode 100644 (file)
index 0000000..778fe58
--- /dev/null
+++ b/lib/leakybucket.php
@@ -0,0 +1,137 @@
+<?php
+
+       function getBucketMemcache()
+       {
+               static $m;
+
+               if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
+               if (!isset($m))
+               {
+                       $m = new Memcached();
+                       $m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
+               }
+               return $m;
+       }
+
+       function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
+       {
+               $m = getBucketMemcache();
+               if (!$m) return 0;
+
+               $iMaxVal = 0;
+               $t = time();
+
+               foreach($asKey as $sKey)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock)
+                       {
+                               $aCurrentBlock = array($iRequestCost, $t);
+                       }
+                       else
+                       {
+                               // add RequestCost
+                               // remove leak * the time since the last request 
+                               $aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
+                               $aCurrentBlock[1] = $t;
+                       }
+
+                       if ($aCurrentBlock[0] <= 0)
+                       {
+                               $m->delete($sKey);
+                       }
+                       else
+                       {
+                               // If we have hit the threshold stop and record this to the block list
+                               if ($aCurrentBlock[0] >= $iThreshold)
+                               {
+                                       $aCurrentBlock[0] = $iThreshold;
+
+                                       // Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
+                                       $i = 10;
+                                       for($i = 0; $i < 10; $i++)
+                                       {
+                                               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               if (!$aBlockedList)
+                                               {
+                                                       $aBlockedList = array();
+                                                       $m->add('blockedList', $aBlockedList);
+                                                       $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               }
+                                               if (!isset($aBlockedList[$sKey]))
+                                               {
+                                                       $aBlockedList[$sKey] = array(1, $t);
+                                               }
+                                               else
+                                               {
+                                                       $aBlockedList[$sKey][0]++;
+                                                       $aBlockedList[$sKey][1] = $t;
+                                               }
+                                               if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
+                                               {
+                                                       uasort($aBlockedList, 'byValue1');
+                                                       $aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
+                                               }
+                                               $x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
+                                               if ($x) break;
+                                       }
+                               }
+                               // Only keep in memcache until the time it would have expired (to avoid clutering memcache)
+                               $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
+                       }
+
+                       // Bucket result in the largest bucket we find
+                       $iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
+               }
+
+               return $iMaxVal;
+        }
+
+       function byValue1($a, $b)
+       {
+               if ($a[1] == $b[1])
+               {
+                       return 0;
+               }
+               return ($a[1] > $b[1]) ? -1 : 1;
+       }
+
+       function byLastBlockTime($a, $b)
+       {
+               if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
+               {
+                       return 0;
+               }
+               return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
+       }
+
+       function getBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return null;
+               $t = time();
+               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+               if (!$aBlockedList) $aBlockedList = array();
+               foreach($aBlockedList as $sKey => $aDetails)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
+                       $iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
+                       $aBlockedList[$sKey] = array(
+                               'totalBlocks' => $aDetails[0],
+                               'lastBlockTimestamp' => $aDetails[1],
+                               'currentBucketSize' => $iCurrentBucketSize,
+                               'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
+                               );
+               }
+               uasort($aBlockedList, 'byLastBlockTime');
+               return $aBlockedList;
+       }
+
+       function clearBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return false;
+               $m->delete('blockedList');
+               return true;
+       }

diff --git a/settings/settings.php b/settings/settings.php
index 6bf526caaf8e05cadcec65c6a26eed0be0ffbdbe..58463a516c2a28cbe147a92e7f92e56a186f1abd 100644 (file)
--- a/settings/settings.php
+++ b/settings/settings.php
@@ -17,6 +17,7 @@
        // Connection buckets to rate limit people being nasty
        @define('CONST_ConnectionBucket_MemcacheServerAddress', false);
        @define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
        // Connection buckets to rate limit people being nasty
        @define('CONST_ConnectionBucket_MemcacheServerAddress', false);
        @define('CONST_ConnectionBucket_MemcacheServerPort', 11211);

+       @define('CONST_ConnectionBucket_MaxBlockList', 100);

        @define('CONST_ConnectionBucket_LeakRate', 1);
        @define('CONST_ConnectionBucket_BlockLimit', 10);
        @define('CONST_ConnectionBucket_WaitLimit', 6);
        @define('CONST_ConnectionBucket_LeakRate', 1);
        @define('CONST_ConnectionBucket_BlockLimit', 10);
        @define('CONST_ConnectionBucket_WaitLimit', 6);

diff --git a/utils/blocks.php b/utils/blocks.php
index d2db17f0e9478b60e9dbc8e9f9faf8cb3a8e6c1b..6dee2845b06bfeb315da6af32b08c07ae5f7f4e1 100755 (executable)
--- a/utils/blocks.php
+++ b/utils/blocks.php
@@ -11,6 +11,7 @@
                array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
                array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
                array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
                array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
                array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
                array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),

+               array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),

        );
        getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
        );
        getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 


@@ -28,11 +29,13 @@
 
                $aBlocks = getBucketBlocks();
                echo "\n";
 
                $aBlocks = getBucketBlocks();
                echo "\n";

-               printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Req Blocked", "Last Block Time");
-               printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-16s-|-%'-31s\n", "", "", "", "", "", "");
+               printf(" %-40s | %12s | %7s | %13s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time");
+               printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s\n", "", "", "", "", "");

                foreach($aBlocks as $sKey => $aDetails)
                {
                foreach($aBlocks as $sKey => $aDetails)
                {

-                       printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp']));
+                       printf(" %-40s | %12s | %7s | %13s | %31s\n", $sKey, $aDetails['totalBlocks'], 
+                               (int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N', 
+                               date("r", $aDetails['lastBlockTimestamp']));

                }
                echo "\n";
        }
                }
                echo "\n";
        }


@@ -42,3 +45,8 @@
                $m->set('sleepCounter', 0);
                clearBucketBlocks();
        }
                $m->set('sleepCounter', 0);
                clearBucketBlocks();
        }

+
+       if ($aResult['flush'])
+       {
+               $m->flush();
+       }

diff --git a/website/details.php b/website/details.php
index 3d80ea5f7b96215f756790589c34bd65ab5c90b6..9cbbf28f87b94182c99c3eedc6b327f495c0affc 100755 (executable)
--- a/website/details.php
+++ b/website/details.php
@@ -1,4 +1,6 @@
 <?php
 <?php

+       @define('CONST_ConnectionBucket_PageType', 'Details');
+

         require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
         require_once(CONST_BasePath.'/lib/log.php');
 
         require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
         require_once(CONST_BasePath.'/lib/log.php');
 

diff --git a/website/reverse.php b/website/reverse.php
index c155fa573030fa54cb5a71629727b808dc2be972..d5a36998efd4143a7513dc6f5f8a30797f9e4aab 100755 (executable)
--- a/website/reverse.php
+++ b/website/reverse.php
@@ -1,4 +1,6 @@
 <?php
 <?php

+       @define('CONST_ConnectionBucket_PageType', 'Reverse');
+

        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');
 
        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');
 

diff --git a/website/search.php b/website/search.php
index 08c2eee095049c3ff6090c3de03bb42a265e6d8c..e69ea78d4d3ed126af55ef656e6bdf13752c45d4 100755 (executable)
--- a/website/search.php
+++ b/website/search.php
@@ -1,4 +1,6 @@
 <?php
 <?php

+       @define('CONST_ConnectionBucket_PageType', 'Search');
+

        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');
 
        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');