X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/cb908b0a30bbffb3a35f49aa13e249cf3b61b152..6e908fb81c5a1d66e80d71c7b4676cacb1d8e801:/utils/cron_ipanalyse.py diff --git a/utils/cron_ipanalyse.py b/utils/cron_ipanalyse.py index 7a70e586..97bad8da 100755 --- a/utils/cron_ipanalyse.py +++ b/utils/cron_ipanalyse.py @@ -34,6 +34,8 @@ UA_BLOCKLIST = () BLOCKCOOLOFF_DELTA=timedelta(hours=1) # quiet time before an IP is released from the bulk pool BULKCOOLOFF_DELTA=timedelta(minutes=15) +# time to check if new accesses appear despite being blocked +BLOCKCHECK_DELTA=timedelta(minutes=1) BULKLONG_LIMIT=8000 BULKSHORT_LIMIT=2000 @@ -85,7 +87,9 @@ class LogEntry: if qp[0] == 'OPTIONS': self.request = None else: - if '/search' in qp[1]: + if '/?' in qp[1]: + self.request = 'S' + elif '/search' in qp[1]: self.request = 'S' elif '/reverse' in qp[1]: self.request = 'R' @@ -219,14 +223,18 @@ class BlockList: class IPstats: def __init__(self): + self.redirected = 0 self.short_total = 0 self.short_api = 0 self.long_total = 0 self.long_api = 0 + self.block_total = 0 self.bad_ua = False def add_long(self, logentry): self.long_total += 1 + if logentry.retcode == 301: + return if logentry.request is not None: self.long_api += 1 if not self.bad_ua: @@ -235,16 +243,27 @@ class IPstats: def add_short(self, logentry): self.short_total += 1 + if logentry.retcode == 301: + self.redirected += 1 + return if logentry.request is not None: self.short_api += 1 self.add_long(logentry) + def add_block(self, logentry): + self.block_total += 1 + + def ignores_warnings(self, wasblocked): + return self.block_total > 5 or (wasblocked and self.redirected > 5) + def new_state(self, was_blocked, was_bulked): if was_blocked: # deblock only if the IP has been really quiet # (properly catches the ones that simply ignore the HTTP error) return None if self.long_total < 20 else 'block' - if self.long_api > BLOCK_UPPER or self.short_api > BLOCK_UPPER / 3: + if self.long_api > BLOCK_UPPER \ + or self.short_api > BLOCK_UPPER / 3 \ + or (self.redirected > 100 and self.short_total == self.redirected): # client totally overdoing it return 'block' if was_bulked: @@ -285,6 +304,7 @@ if __name__ == '__main__': bl = BlockList() shortstart = dt + BLOCKCOOLOFF_DELTA - BULKCOOLOFF_DELTA + blockstart = dt + BLOCKCOOLOFF_DELTA - BLOCKCHECK_DELTA notlogged = bl.whitelist | bl.blacklist stats = defaultdict(IPstats) @@ -301,6 +321,8 @@ if __name__ == '__main__': stats[l.ip].add_short(l) if l.request is not None and l.retcode == 200: total200 += 1 + if l.date > blockstart and l.retcode in (403, 429): + stats[l.ip].add_block(l) # adapt limits according to CPU and DB load fd = open("/proc/loadavg") @@ -360,7 +382,7 @@ if __name__ == '__main__': fd.close() # TODO write logs (need to collect some statistics) - logstr = datetime.now().strftime('%Y-%m-%d %H:%M') + ' %s %s\n' + logstr = datetime.now().strftime('%d/%b/%Y:%H:%M:%S') + ' %s %s\n' fd = open(LOGFILE, 'a') if unblocked: fd.write(logstr % ('unblocked:', ', '.join(unblocked))) @@ -374,4 +396,7 @@ if __name__ == '__main__': fd.write(logstr % (' ua block:', ', '.join(uablocked))) if blocked: fd.write(logstr % ('new block:', ', '.join(blocked))) + #for k,v in stats.items(): + # if v.ignores_warnings(k in bl.prevblocks) and k not in notlogged and ':' not in k: + # fd.write(logstr % ('Warning ignored:', k)) fd.close()