import re
import os
import sys
+import subprocess
from datetime import datetime, timedelta
from collections import defaultdict
UA_BLOCKLIST = ()
# time before a automatically blocked IP is allowed back
-BLOCKCOOLOFF_PERIOD=timedelta(hours=1)
+BLOCKCOOLOFF_DELTA=timedelta(hours=1)
# quiet time before an IP is released from the bulk pool
-BULKCOOLOFF_PERIOD=timedelta(minutes=15)
+BULKCOOLOFF_DELTA=timedelta(minutes=15)
BULKLONG_LIMIT=8000
BULKSHORT_LIMIT=2000
try:
with open(BASEDIR + "/settings/ip_blocks.conf") as f:
code = compile(f.read(), BASEDIR + "/settings/ip_blocks.conf", 'exec')
- exec(code, global_vars, local_vars)
+ exec(code)
except IOError:
pass
BLOCK_LIMIT = BLOCK_LOWER
+time_regex = r'(?P<t_day>\d\d)/(?P<t_month>[A-Za-z]+)/(?P<t_year>\d\d\d\d):(?P<t_hour>\d\d):(?P<t_min>\d\d):(?P<t_sec>\d\d) [+-]\d\d\d\d'
-format_pat= re.compile('(?P<ip>[(\d\.)]+) - - \[(?P<time>.*?)\] "(?P<query>.*?)" (?P<return>\d+) (?P<bytes>\d+) "(?P<referer>.*?)" "(?P<ua>.*?)"')
-time_pat= re.compile('[(\d\.)]+ - - \[(?P<time>[^\]]*?)\] ')
+format_pat= re.compile(r'(?P<ip>[a-f\d\.:]+) - - \['+ time_regex + r'] "(?P<query>.*?)" (?P<return>\d+) (?P<bytes>\d+) "(?P<referer>.*?)" "(?P<ua>.*?)"')
+time_pat= re.compile(r'[a-f\d:\.]+ - - \[' + time_regex + '\] ')
logtime_pat = "%d/%b/%Y:%H:%M:%S %z"
+MONTHS = { 'Jan' : 1, 'Feb' : 2, 'Mar' : 3, 'Apr' : 4, 'May' : 5, 'Jun' : 6,
+ 'Jul' : 7, 'Aug' : 8, 'Sep' : 9, 'Oct' : 10, 'Nov' : 11, 'Dec' : 12 }
+
class LogEntry:
def __init__(self, logline):
e = format_pat.match(logline)
raise ValueError("Invalid log line:", logline)
e = e.groupdict()
self.ip = e['ip']
- self.date = datetime.strptime(e['time'], logtime_pat).replace(tzinfo=None)
+ self.date = datetime(int(e['t_year']), MONTHS[e['t_month']], int(e['t_day']),
+ int(e['t_hour']), int(e['t_min']), int(e['t_sec']))
qp = e['query'].split(' ', 2)
if len(qp) < 2:
self.request = None
if qp[0] == 'OPTIONS':
self.request = None
else:
- if qp[1].startswith('/search'):
+ if '/search' in qp[1]:
self.request = 'S'
- elif qp[1].startswith('/reverse'):
+ elif '/reverse' in qp[1]:
self.request = 'R'
- elif qp[1].startswith('/details'):
+ elif '/details' in qp[1]:
self.request = 'D'
else:
self.request = None
if e is None:
return None
e = e.groupdict()
- return datetime.strptime(e['time'], logtime_pat).replace(tzinfo=None)
+ #return datetime.strptime(e['time'], logtime_pat).replace(tzinfo=None)
+ return datetime(int(e['t_year']), MONTHS[e['t_month']], int(e['t_day']),
+ int(e['t_hour']), int(e['t_min']), int(e['t_sec']))
class LogFile:
return LogEntry.get_log_time(l) if l is not None else None
def seek_to_date(self, target):
- date1 = self.seek_next(0)
- if date1 > target:
+ # start position for binary search
+ fromseek = 0
+ fromdate = self.seek_next(0)
+ if fromdate > target:
+ return True
+ # end position for binary search
+ toseek = -100
+ while -toseek < self.len:
+ todate = self.seek_next(self.len + toseek)
+ if todate is not None:
+ break
+ toseek -= 100
+ if todate is None or todate < target:
return False
- curseek = 2*1024*1024
- curdate = self.seek_next(curseek)
- if curdate is None:
- raise RuntimeError("Cannot seek to %d" % curseek)
- while abs((target - curdate).total_seconds()) > 1.0:
- bps = curseek / ((curdate - date1).total_seconds())
- curseek += (target - curdate).total_seconds() * bps
- if curseek < 0:
- curseek = 0
- elif curseek > self.len:
- curseek = self.len - bps
- curdate = self.seek_next(curseek)
- if curdate is None:
- raise RuntimeError("Cannot see to %d" % curseek)
- return True
+ toseek = self.len + toseek
+
+
+ while True:
+ bps = (toseek - fromseek) / (todate - fromdate).total_seconds()
+ newseek = fromseek + int((target - fromdate).total_seconds() * bps)
+ newdate = self.seek_next(newseek)
+ if newdate is None:
+ return False;
+ error = abs((target - newdate).total_seconds())
+ if error < 1:
+ return True
+ if newdate > target:
+ toseek = newseek
+ todate = newdate
+ oldfromseek = fromseek
+ fromseek = toseek - error * bps
+ while True:
+ if fromseek <= oldfromseek:
+ fromseek = oldfromseek
+ fromdate = self.seek_next(fromseek)
+ break
+ fromdate = self.seek_next(fromseek)
+ if fromdate < target:
+ break;
+ bps *=2
+ fromseek -= error * bps
+ else:
+ fromseek = newseek
+ fromdate = newdate
+ oldtoseek = toseek
+ toseek = fromseek + error * bps
+ while True:
+ if toseek > oldtoseek:
+ toseek = oldtoseek
+ todate = self.seek_next(toseek)
+ break
+ todate = self.seek_next(toseek)
+ if todate > target:
+ break
+ bps *=2
+ toseek += error * bps
+ if toseek - fromseek < 500:
+ return True
+
def loglines(self):
for l in self.fd:
- yield LogEntry(l)
+ try:
+ yield LogEntry(l)
+ except ValueError:
+ pass # ignore invalid lines
class BlockList:
ip, typ = line.strip().split(' ')
if ip not in self.blacklist:
if typ == 'block':
- prevblocks.add(ip)
+ self.prevblocks.add(ip)
elif typ == 'bulk':
- prevbulks.add(ip)
+ self.prevbulks.add(ip)
fd.close()
except IOError:
pass #ignore non-existing file
sys.exit(-1)
if len(sys.argv) == 2:
- dt = datetime.now() - BLOCKCOOLOFF_PERIOD
+ dt = datetime.now() - BLOCKCOOLOFF_DELTA
else:
dt = datetime.strptime(sys.argv[2], "%Y-%m-%d %H:%M:%S")
bl = BlockList()
- shortstart = dt + BLOCKCOOLOFF_PERIOD - BULKCOOLOFF_PERIOD
+ shortstart = dt + BLOCKCOOLOFF_DELTA - BULKCOOLOFF_DELTA
notlogged = bl.whitelist | bl.blacklist
stats = defaultdict(IPstats)
fd = open("/proc/loadavg")
cpuload = int(float(fd.readline().split()[2]))
fd.close()
- dbload = total200 / BULKCOOLOFF_PERIOD.total_seconds()
+ # check the number of excess connections to apache
+ dbcons = int(subprocess.check_output("netstat -s | grep 'connections established' | sed 's:^\s*::;s: .*::'", shell=True))
+ fpms = int(subprocess.check_output('ps -Af | grep php-fpm | wc -l', shell=True))
+ dbload = max(0, dbcons - fpms)
numbulks = len(bl.prevbulks)
- BLOCK_LIMIT = max(BLOCK_LIMIT, BLOCK_UPPER - BLOCK_LOADFAC * (dbload - 75))
- BULKLONG_LIMIT = max(BULK_LOWER, BULKLONG_LIMIT - BULK_LOADFAC * (cpuload - 14))
+ BLOCK_LIMIT = max(BLOCK_LIMIT, BLOCK_UPPER - BLOCK_LOADFAC * dbload)
+ BULKLONG_LIMIT = max(BULK_LOWER, BULKLONG_LIMIT - BULK_LOADFAC * cpuload)
if numbulks > MAX_BULK_IPS:
BLOCK_LIMIT = max(3600, BLOCK_LOWER - (numbulks - MAX_BULK_IPS)*10)
# if the bulk pool is still empty, clients will be faster, avoid having
elif wasbulked:
debulked.append(k)
for i in bl.blacklist:
- fd.write("%s ban\n" % k)
+ fd.write("%s ban\n" % i)
fd.close()
# TODO write logs (need to collect some statistics)
projects / nominatim.git / blobdiff