From a049e8a7c6745513487e5811caf2ae3dd92d3a37 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 12 May 2012 17:20:27 +0200 Subject: [PATCH] automated banning of excessive bulk users --- lib/init-website.php | 46 +++++++++++++++--------- lib/init.php | 1 + settings/settings.php | 4 +-- utils/cron_banip.sh | 84 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 19 deletions(-) create mode 100755 utils/cron_banip.sh diff --git a/lib/init-website.php b/lib/init-website.php index 36928628..6db2ac37 100644 --- a/lib/init-website.php +++ b/lib/init-website.php @@ -1,22 +1,34 @@

Access blocked

"; - echo "Your IP has been blocked for overusing OpenStreetMap's volunteer-run servers.
\n"; - echo 'Please consult the Nominatim usage policy for more information.'; - echo "\n\n"; - exit; - } + if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) + { + header('HTTP/1.0 403 Forbidden'); + header('Content-type: text/html; charset=utf-8'); + echo "

Access blocked

"; + echo "Your IP has been blocked for overusing OpenStreetMap's volunteer-run servers.
\n"; + echo 'Please consult the Nominatim usage policy for more information.'; + echo "\n\n"; + exit; + } - header('Content-type: text/html; charset=utf-8'); + $sTempBlockedIP = file_get_contents(CONST_IPBanFile); + if (preg_match('/\b'.$_SERVER["REMOTE_ADDR"].'\b/', $sTempBlockedIP)) + { + header('HTTP/1.0 503 Service Temporarily Unavailable'); + header('Content-type: text/html; charset=utf-8'); + echo "

Access blocked

"; + echo "Your IP has been blocked temporarily for overusing OpenStreetMap's volunteer-run servers. This ban will be lifted automatically in a while. To avoid further blocks, please read the
\n"; + echo 'Nominatim usage policy carefully before you continue to use this service.'; + echo "\n\n"; + exit; + } + + } diff --git a/lib/init.php b/lib/init.php index 15e38a5b..18fbc9a6 100644 --- a/lib/init.php +++ b/lib/init.php @@ -1,6 +1,7 @@ $BLOCKEDFILE.newblocks << ENDOFQUERY +SELECT ipaddress FROM +((SELECT ipaddress FROM + (SELECT ipaddress, count(*) FROM new_query_log + WHERE type = 'reverse' AND starttime > now() - interval '$LONG_PERIOD' + GROUP BY ipaddress) + as v + WHERE count > $REVLONG_LIMIT) +UNION +(SELECT ipaddress FROM + (SELECT ipaddress, count(*) FROM new_query_log + WHERE type = 'reverse' AND starttime > now() - interval '$SHORT_PERIOD' + GROUP BY ipaddress) + as v + WHERE count > $REVSHORT_LIMIT) +UNION +(SELECT ipaddress FROM + (SELECT ipaddress, count(*) FROM new_query_log + WHERE type = 'search' AND starttime > now() - interval '$LONG_PERIOD' + GROUP BY ipaddress) + as v + WHERE count > $SRCHLONG_LIMIT) +UNION +(SELECT ipaddress FROM + (SELECT ipaddress, sum(endtime-starttime) as dur FROM new_query_log + WHERE type = 'search' AND starttime > now() - interval '$SHORT_PERIOD' + GROUP BY ipaddress) + as v + WHERE dur > '$SRCHSHORT_LIMIT') +) as q ORDER BY ipaddress; +ENDOFQUERY + +no_newblocks=`comm $BLOCKEDFILE.newblocks $BLOCKEDFILE -23 | wc -l` + +if [ "x$no_newblocks" != "x0" ]; then + date +"%x %X Newly blocked IPs: `comm $BLOCKEDFILE.newblocks $BLOCKEDFILE -23 | tr '\n' ' '`" >> $LOGFILE +fi + + +# Deblockable candidates +blocked=`tr '\n' ',' < $BLOCKEDFILE | sed "s:[[:space:]]::g;s:,$::;s:,:'),(':g"` + +if [ "x$blocked" == "x" ]; then + mv $BLOCKEDFILE.newblocks $BLOCKEDFILE +else + $PSQLCMD > $BLOCKEDFILE.newlifted << ENDOFQUERY + VALUES ('$blocked') + EXCEPT + (SELECT DISTINCT ipaddress FROM new_query_log + WHERE starttime > now() - interval '$COOLOFF_PERIOD') +ENDOFQUERY + + no_lifted=`cat $BLOCKEDFILE.newlifted | wc -w` + + if [ "x$no_lifted" != "x0" ]; then + date +"%x %X Bans lifted: `tr '\n' ' ' < $BLOCKEDFILE.newlifted`" >> $LOGFILE + fi + + # Write out new blocks + cat $BLOCKEDFILE.newblocks $BLOCKEDFILE | sort -u | comm - $BLOCKEDFILE.newlifted -23 > $BLOCKEDFILE.new + mv $BLOCKEDFILE.new $BLOCKEDFILE + + rm $BLOCKEDFILE.newblocks $BLOCKEDFILE.newlifted +fi -- 2.39.5