rate limiting using memcache

diff --git a/lib/init-website.php b/lib/init-website.php
index d21ca7af94bd14c125b135e7a1a436d5fec03578..896a7b24293e86d68c70e5524544aa9a9197e7e4 100644 (file)
--- a/lib/init-website.php
+++ b/lib/init-website.php
@@ -1,5 +1,8 @@
 <?php
-
+//echo "<a href=\"http://localhost/nominatim/search.php?format=xml&addressdetails=1&accept-language=en&street=&suburb=&city=Dammam&county=&state=&country=SA&postcode=&\">x</a>";
+//exit;
+//phpinfo();
+//exit;
        require_once('init.php');
 
        if (CONST_ClosedForIndexing && strpos(CONST_ClosedForIndexingExceptionIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
@@ -8,7 +11,16 @@
                exit;
        }
 
-       if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
+       $aBucketKeys = array();
+       if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
+       if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"];
+       if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"];
+
+       $fBucketVal = doBucket($aBucketKeys, 
+                       (defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(),
+                       CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+
+       if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
        {
                echo "Your IP has been blocked. \n";
                echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n";
@@ -16,4 +28,10 @@
                exit;
        }
 
+       if ($fBucketVal > CONST_ConnectionBucket_WaitLimit)
+       {
+               sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate);
+       }
+var_dump($fBucketVal);
+exit;
        header('Content-type: text/html; charset=utf-8');

diff --git a/lib/lib.php b/lib/lib.php
index f34b09b7a51fbef8285d4a19bd9fb9de5b1704ab..20f5af17787c7386cd98594f74c1e5c30198ead5 100644 (file)
--- a/lib/lib.php
+++ b/lib/lib.php
@@ -60,7 +60,7 @@
        {
                $sLoadAverage = file_get_contents('/proc/loadavg');
                 $aLoadAverage = explode(' ',$sLoadAverage);
-               return (int)$aLoadAverage[0];
+               return (float)$aLoadAverage[0];
        }
 
        function getProcessorCount()
@@ -917,3 +917,110 @@
                 return true;
         }
 
+       function getBucketMemcache()
+       {
+               if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
+               $m = new Memcached();
+               $m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
+               return $m;
+       }
+
+       function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
+       {
+               $m = getBucketMemcache();
+               if (!$m) return 0;
+
+               $iMaxVal = 0;
+               $t = time();
+
+               foreach($asKey as $sKey)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock)
+                       {
+                               $aCurrentBlock = array($iRequestCost, $t);
+                       }
+                       else
+                       {
+                               // add RequestCost
+                               // remove leak * the time since the last request 
+                               $aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
+                               $aCurrentBlock[1] = $t;
+                       }
+
+                       if ($aCurrentBlock[0] <= 0)
+                       {
+                               $m->delete($sKey);
+                       }
+                       else
+                       {
+                               // If we have hit the threshold stop and record this to the block list
+                               if ($aCurrentBlock[0] >= $iThreshold)
+                               {
+                                       $aCurrentBlock[0] = $iThreshold;
+
+                                       // Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
+                                       $i = 10;
+                                       for($i = 0; $i < 10; $i++)
+                                       {
+                                               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               if (!$aBlockedList)
+                                               {
+                                                       $aBlockedList = array();
+                                                       $m->add('blockedList', $aBlockedList);
+                                                       $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               }
+                                               if (!isset($aBlockedList[$sKey]))
+                                               {
+                                                       $aBlockedList[$sKey] = array(1, $t);
+                                               }
+                                               else
+                                               {
+                                                       $aBlockedList[$sKey][0]++;
+                                                       $aBlockedList[$sKey][1] = $t;
+                                               }
+                                               $x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
+                                               if ($x) break;
+                                       }
+                               }
+                               // Only keep in memcache until the time it would have expired (to avoid clutering memcache)
+                               $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
+                       }
+
+                       // Bucket result in the largest bucket we find
+                       $iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
+               }
+
+               return $iMaxVal;
+        }
+
+       function getBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return null;
+               $t = time();
+               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+               if (!$aBlockedList) $aBlockedList = array();
+               foreach($aBlockedList as $sKey => $aDetails)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
+                       $iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
+                       $aBlockedList[$sKey] = array(
+                               'totalBlocks' => $aDetails[0],
+                               'lastBlockTimestamp' => $aDetails[1],
+                               'currentBucketSize' => $iCurrentBucketSize,
+                               'lastRequestBlocked' => $aCurrentBlock[0] >= CONST_ConnectionBucket_BlockLimit,
+                               'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
+                               );
+               }
+               return $aBlockedList;
+       }
+
+       function clearBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return false;
+               $m->delete('blockedList');
+               return true;
+       }

diff --git a/settings/settings.php b/settings/settings.php
index 0a214ce9fa92746432072a5ee848d0b17b69a6ce..656040358e1bef60018bc03347206eb0cb80c5e2 100644 (file)
--- a/settings/settings.php
+++ b/settings/settings.php
@@ -14,6 +14,24 @@
        @define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
        @define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
 
+       // Connection buckets to rate limit people being nasty
+       @define('CONST_ConnectionBucket_MemcacheServerAddress', false);
+       @define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
+       @define('CONST_ConnectionBucket_LeakRate', 1);
+       @define('CONST_ConnectionBucket_BlockLimit', 10);
+       @define('CONST_ConnectionBucket_WaitLimit', 6);
+       @define('CONST_ConnectionBucket_Cost_Reverse', 1);
+       @define('CONST_ConnectionBucket_Cost_Search', 2);
+       @define('CONST_ConnectionBucket_Cost_Details', 3);
+
+       if (!function_exists('user_busy_cost'))
+       {
+               function user_busy_cost()
+               {
+                       return 0;
+               }
+       }
+
        // Website settings
        @define('CONST_ClosedForIndexing', false);
        @define('CONST_ClosedForIndexingExceptionIPs', '');
@@ -30,10 +48,11 @@
 
        @define('CONST_Search_AreaPolygons_Enabled', true);
        @define('CONST_Search_AreaPolygons', true);
-       @define('CONST_Search_TryDroppedAddressTerms', false);
 
        @define('CONST_Suggestions_Enabled', false);
 
+       @define('CONST_Search_TryDroppedAddressTerms', false);
+
        // Set to zero to disable polygon output
        @define('CONST_PolygonOutput_MaximumTypes', 1);
 

diff --git a/utils/blocks.php b/utils/blocks.php
new file mode 100755 (executable)
index 0000000..4d32311
--- /dev/null
+++ b/utils/blocks.php
@@ -0,0 +1,40 @@
+#!/usr/bin/php -Cq
+<?php
+
+        require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
+        ini_set('memory_limit', '800M');
+
+       $aCMDOptions = array(
+               "Manage service blocks / restrictions",
+               array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+               array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+               array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+               array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
+               array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
+       );
+       getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
+
+       $m = getBucketMemcache();
+        if (!$m)
+       {
+               echo "ERROR: Bucket memcache is not configured\n";
+               exit;
+       }
+
+       if ($aResult['list'])
+       {
+               $aBlocks = getBucketBlocks();
+               echo "\n";
+               printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Req Blocked", "Last Block Time");
+               printf(" %'--40s | %'-12s | %'-7s | %'-13s | %'-16s | %'-31s\n", "", "", "", "", "", "");
+               foreach($aBlocks as $sKey => $aDetails)
+               {
+                       printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp']));
+               }
+               echo "\n";
+       }
+
+       if ($aResult['delete'])
+       {
+               clearBucketBlocks();
+       }