]> git.openstreetmap.org Git - nominatim.git/commitdiff
projects / nominatim.git / commitdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: d992358)
automated banning of excessive bulk users
authorSarah Hoffmann <lonvia@denofr.de>
Sat, 12 May 2012 15:20:27 +0000 (17:20 +0200)
committerSarah Hoffmann <lonvia@denofr.de>
Sat, 12 May 2012 15:20:27 +0000 (17:20 +0200)
lib/init-website.php patch | blob | history
lib/init.php patch | blob | history
settings/settings.php patch | blob | history
utils/cron_banip.sh [new file with mode: 0755] patch | blob

diff --git a/lib/init-website.php b/lib/init-website.php
index 36928628611b228d5651312a5aaa3432055da3dc..6db2ac374d995837b2aac2dcb9861ec6922945b6 100644 (file)
--- a/lib/init-website.php
+++ b/lib/init-website.php
@@ -1,22 +1,34 @@
 <?php
 <?php
+    require_once('init.php');
 
 
-       require_once('init.php');
+    header('Content-type: text/html; charset=utf-8');
 
 
-       if (CONST_ClosedForIndexing && strpos(CONST_ClosedForIndexingExceptionIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
-       {
-               echo "Closed for re-indexing...";
-               exit;
-       }
+    // check blocks in place for external servers
+    if (strpos($_SERVER["REMOTE_ADDR"],'193.63.75.') !== 0 &&
+        strpos(CONST_WhitelistedIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
+    {
 
 
-       if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
-       {
-               header('HTTP/1.0 403 Forbidden');
-               header('Content-type: text/html; charset=utf-8');
-               echo "<html><body><h1>Access blocked</h1>";
-               echo "Your IP has been blocked for overusing OpenStreetMap's volunteer-run servers.<br> \n";
-               echo 'Please consult the <a href="http://wiki.openstreetmap.org/wiki/Nominatim_usage_policy">Nominatim usage policy</a> for more information.';
-               echo "\n</body></html>\n";
-               exit;
-       }
+        if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
+        {
+            header('HTTP/1.0 403 Forbidden');
+            header('Content-type: text/html; charset=utf-8');
+            echo "<html><body><h1>Access blocked</h1>";
+            echo "Your IP has been blocked for overusing OpenStreetMap's volunteer-run servers.<br> \n";
+            echo 'Please consult the <a href="http://wiki.openstreetmap.org/wiki/Nominatim_usage_policy">Nominatim usage policy</a> for more information.';
+            echo "\n</body></html>\n";
+            exit;
+        }
 
 
-       header('Content-type: text/html; charset=utf-8');
+        $sTempBlockedIP = file_get_contents(CONST_IPBanFile);
+        if (preg_match('/\b'.$_SERVER["REMOTE_ADDR"].'\b/', $sTempBlockedIP))
+        {
+            header('HTTP/1.0 503 Service Temporarily Unavailable');
+            header('Content-type: text/html; charset=utf-8');
+            echo "<html><body><h1>Access blocked</h1>";
+            echo "Your IP has been blocked temporarily for overusing OpenStreetMap's volunteer-run servers. This ban will be lifted automatically in a while. To avoid further blocks, please read the<br> \n";
+            echo '<a href="http://wiki.openstreetmap.org/wiki/Nominatim_usage_policy">Nominatim usage policy</a> carefully before you continue to use this service.';
+            echo "\n</body></html>\n";
+            exit;
+        }
+
+    }
diff --git a/lib/init.php b/lib/init.php
index 15e38a5b17fcf7862e71bacda260db87e7b03d60..18fbc9a688259440a3b92b42e9e1d61ccd9608fe 100644 (file)
--- a/lib/init.php
+++ b/lib/init.php
@@ -1,6 +1,7 @@
 <?php
 
        @define('CONST_BasePath', dirname(dirname(__FILE__)));
 <?php
 
        @define('CONST_BasePath', dirname(dirname(__FILE__)));
+       date_default_timezone_set('UTC');
 
        require_once(CONST_BasePath.'/settings/settings.php');
        require_once(CONST_BasePath.'/lib/lib.php');
 
        require_once(CONST_BasePath.'/settings/settings.php');
        require_once(CONST_BasePath.'/lib/lib.php');
diff --git a/settings/settings.php b/settings/settings.php
index edbe26048cc61439634e0353764402b68c8acc31..ef2ebe4ee6de6afea753329fe55dcfe5b46c36b3 100644 (file)
--- a/settings/settings.php
+++ b/settings/settings.php
@@ -14,9 +14,9 @@
     @define('CONST_Osmosis_Binary', CONST_BasePath.'/../osmosis-0.40.1/bin/osmosis');
 
        // Website settings
     @define('CONST_Osmosis_Binary', CONST_BasePath.'/../osmosis-0.40.1/bin/osmosis');
 
        // Website settings
-       @define('CONST_ClosedForIndexing', false);
-       @define('CONST_ClosedForIndexingExceptionIPs', '');
        @define('CONST_BlockedIPs', '');
        @define('CONST_BlockedIPs', '');
+       @define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks');
+       @define('CONST_WhitelistedIPs', '');
        @define('CONST_BlockedUserAgents', '');
        @define('CONST_BlockReverseMaxLoad', 15);
 
        @define('CONST_BlockedUserAgents', '');
        @define('CONST_BlockReverseMaxLoad', 15);
 
diff --git a/utils/cron_banip.sh b/utils/cron_banip.sh
new file mode 100755 (executable)
index 0000000..1f0aade
--- /dev/null
+++ b/utils/cron_banip.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# Create or update the list of temporarily banned IPs.
+#
+
+BLOCKEDFILE=/home/lonvia/nominatim/settings/ip_blocks
+LOGFILE=/home/lonvia/nominatim/log/ip_blocks.log
+
+LONG_PERIOD='1 hour'
+SHORT_PERIOD='10 min'
+COOLOFF_PERIOD='1 hour'
+
+REVLONG_LIMIT=20000
+REVSHORT_LIMIT=6000
+SRCHLONG_LIMIT=4000
+SRCHSHORT_LIMIT='10 min'
+
+PSQLCMD='psql -qtA -d nominatim'
+
+# Blocking candidates
+$PSQLCMD > $BLOCKEDFILE.newblocks << ENDOFQUERY
+SELECT ipaddress FROM
+((SELECT ipaddress FROM
+  (SELECT ipaddress, count(*) FROM new_query_log
+   WHERE type = 'reverse' AND starttime > now() - interval '$LONG_PERIOD'
+   GROUP BY ipaddress)
+  as v
+  WHERE count > $REVLONG_LIMIT) 
+UNION
+(SELECT ipaddress FROM
+  (SELECT ipaddress, count(*) FROM new_query_log
+   WHERE type = 'reverse' AND starttime > now() - interval '$SHORT_PERIOD'
+   GROUP BY ipaddress)
+  as v
+  WHERE count > $REVSHORT_LIMIT) 
+UNION
+(SELECT ipaddress FROM
+  (SELECT ipaddress, count(*) FROM new_query_log
+   WHERE type = 'search' AND starttime > now() - interval '$LONG_PERIOD'
+   GROUP BY ipaddress)
+  as v
+  WHERE count > $SRCHLONG_LIMIT) 
+UNION
+(SELECT ipaddress FROM
+  (SELECT ipaddress, sum(endtime-starttime) as dur FROM new_query_log
+   WHERE type = 'search' AND starttime > now() - interval '$SHORT_PERIOD'
+   GROUP BY ipaddress)
+  as v
+  WHERE dur > '$SRCHSHORT_LIMIT')
+) as q ORDER BY ipaddress;
+ENDOFQUERY
+
+no_newblocks=`comm $BLOCKEDFILE.newblocks $BLOCKEDFILE -23 | wc -l`
+
+if [ "x$no_newblocks" != "x0" ]; then
+    date +"%x %X Newly blocked IPs: `comm $BLOCKEDFILE.newblocks $BLOCKEDFILE -23 | tr '\n' ' '`" >> $LOGFILE
+fi
+
+
+# Deblockable candidates
+blocked=`tr '\n' ',' < $BLOCKEDFILE | sed "s:[[:space:]]::g;s:,$::;s:,:'),(':g"`
+
+if [ "x$blocked" == "x" ]; then
+  mv $BLOCKEDFILE.newblocks $BLOCKEDFILE 
+else
+    $PSQLCMD > $BLOCKEDFILE.newlifted << ENDOFQUERY
+    VALUES ('$blocked')
+    EXCEPT
+    (SELECT DISTINCT ipaddress FROM new_query_log
+     WHERE starttime > now() - interval '$COOLOFF_PERIOD')
+ENDOFQUERY
+
+    no_lifted=`cat $BLOCKEDFILE.newlifted | wc -w`
+
+    if [ "x$no_lifted" != "x0" ]; then
+        date +"%x %X Bans lifted: `tr '\n' ' ' < $BLOCKEDFILE.newlifted`" >> $LOGFILE
+    fi
+
+    # Write out new blocks
+    cat $BLOCKEDFILE.newblocks $BLOCKEDFILE | sort -u | comm - $BLOCKEDFILE.newlifted -23 > $BLOCKEDFILE.new
+    mv $BLOCKEDFILE.new $BLOCKEDFILE
+
+    rm $BLOCKEDFILE.newblocks $BLOCKEDFILE.newlifted
+fi
Open Source search based on OpenStreetMap data