X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/62dbbbede6ac6b75d0bd75a0d21016bcdfc61742..554d25e2e508c57fe763cee95b574b8e7c66eb0b:/utils/cron_ipanalyse.py?ds=inline diff --git a/utils/cron_ipanalyse.py b/utils/cron_ipanalyse.py index be5e6956..97bad8da 100755 --- a/utils/cron_ipanalyse.py +++ b/utils/cron_ipanalyse.py @@ -34,6 +34,8 @@ UA_BLOCKLIST = () BLOCKCOOLOFF_DELTA=timedelta(hours=1) # quiet time before an IP is released from the bulk pool BULKCOOLOFF_DELTA=timedelta(minutes=15) +# time to check if new accesses appear despite being blocked +BLOCKCHECK_DELTA=timedelta(minutes=1) BULKLONG_LIMIT=8000 BULKSHORT_LIMIT=2000 @@ -221,14 +223,18 @@ class BlockList: class IPstats: def __init__(self): + self.redirected = 0 self.short_total = 0 self.short_api = 0 self.long_total = 0 self.long_api = 0 + self.block_total = 0 self.bad_ua = False def add_long(self, logentry): self.long_total += 1 + if logentry.retcode == 301: + return if logentry.request is not None: self.long_api += 1 if not self.bad_ua: @@ -237,16 +243,27 @@ class IPstats: def add_short(self, logentry): self.short_total += 1 + if logentry.retcode == 301: + self.redirected += 1 + return if logentry.request is not None: self.short_api += 1 self.add_long(logentry) + def add_block(self, logentry): + self.block_total += 1 + + def ignores_warnings(self, wasblocked): + return self.block_total > 5 or (wasblocked and self.redirected > 5) + def new_state(self, was_blocked, was_bulked): if was_blocked: # deblock only if the IP has been really quiet # (properly catches the ones that simply ignore the HTTP error) return None if self.long_total < 20 else 'block' - if self.long_api > BLOCK_UPPER or self.short_api > BLOCK_UPPER / 3: + if self.long_api > BLOCK_UPPER \ + or self.short_api > BLOCK_UPPER / 3 \ + or (self.redirected > 100 and self.short_total == self.redirected): # client totally overdoing it return 'block' if was_bulked: @@ -287,6 +304,7 @@ if __name__ == '__main__': bl = BlockList() shortstart = dt + BLOCKCOOLOFF_DELTA - BULKCOOLOFF_DELTA + blockstart = dt + BLOCKCOOLOFF_DELTA - BLOCKCHECK_DELTA notlogged = bl.whitelist | bl.blacklist stats = defaultdict(IPstats) @@ -303,6 +321,8 @@ if __name__ == '__main__': stats[l.ip].add_short(l) if l.request is not None and l.retcode == 200: total200 += 1 + if l.date > blockstart and l.retcode in (403, 429): + stats[l.ip].add_block(l) # adapt limits according to CPU and DB load fd = open("/proc/loadavg") @@ -362,7 +382,7 @@ if __name__ == '__main__': fd.close() # TODO write logs (need to collect some statistics) - logstr = datetime.now().strftime('%Y-%m-%d %H:%M') + ' %s %s\n' + logstr = datetime.now().strftime('%d/%b/%Y:%H:%M:%S') + ' %s %s\n' fd = open(LOGFILE, 'a') if unblocked: fd.write(logstr % ('unblocked:', ', '.join(unblocked))) @@ -376,4 +396,7 @@ if __name__ == '__main__': fd.write(logstr % (' ua block:', ', '.join(uablocked))) if blocked: fd.write(logstr % ('new block:', ', '.join(blocked))) + #for k,v in stats.items(): + # if v.ignores_warnings(k in bl.prevblocks) and k not in notlogged and ':' not in k: + # fd.write(logstr % ('Warning ignored:', k)) fd.close()