Merge remote-tracking branch 'upstream/master'

[nominatim.git] / utils / cron_ipanalyse.py

diff --git a/utils/cron_ipanalyse.py b/utils/cron_ipanalyse.py
index 4625de707ee24b99ffb2eb4d2b3d2ecd41fd7699..97bad8da118e96b1f4287873919287389a3e2cd8 100755 (executable)
--- a/utils/cron_ipanalyse.py
+++ b/utils/cron_ipanalyse.py
@@ -223,6 +223,7 @@ class BlockList:
 class IPstats:
 
     def __init__(self):
 class IPstats:
 
     def __init__(self):

+        self.redirected = 0

         self.short_total = 0
         self.short_api = 0
         self.long_total = 0
         self.short_total = 0
         self.short_api = 0
         self.long_total = 0


@@ -232,6 +233,8 @@ class IPstats:
 
     def add_long(self, logentry):
         self.long_total += 1
 
     def add_long(self, logentry):
         self.long_total += 1

+        if logentry.retcode == 301:
+            return

         if logentry.request is not None:
             self.long_api += 1
         if not self.bad_ua:
         if logentry.request is not None:
             self.long_api += 1
         if not self.bad_ua:


@@ -240,6 +243,9 @@ class IPstats:
 
     def add_short(self, logentry):
         self.short_total += 1
 
     def add_short(self, logentry):
         self.short_total += 1

+        if logentry.retcode == 301:
+            self.redirected += 1
+            return

         if logentry.request is not None:
             self.short_api += 1
         self.add_long(logentry)
         if logentry.request is not None:
             self.short_api += 1
         self.add_long(logentry)


@@ -247,15 +253,17 @@ class IPstats:
     def add_block(self, logentry):
         self.block_total += 1
 
     def add_block(self, logentry):
         self.block_total += 1
 

-    def ignores_warnings(self):
-        return self.block_total > 5
+    def ignores_warnings(self, wasblocked):
+        return self.block_total > 5 or (wasblocked and self.redirected > 5)

 
     def new_state(self, was_blocked, was_bulked):
         if was_blocked:
             # deblock only if the IP has been really quiet
             # (properly catches the ones that simply ignore the HTTP error)
             return None if self.long_total < 20 else 'block'
 
     def new_state(self, was_blocked, was_bulked):
         if was_blocked:
             # deblock only if the IP has been really quiet
             # (properly catches the ones that simply ignore the HTTP error)
             return None if self.long_total < 20 else 'block'

-        if self.long_api > BLOCK_UPPER or self.short_api > BLOCK_UPPER / 3:
+        if self.long_api > BLOCK_UPPER \
+            or self.short_api > BLOCK_UPPER / 3 \
+            or (self.redirected > 100 and self.short_total == self.redirected):

                 # client totally overdoing it
                 return 'block'
         if was_bulked:
                 # client totally overdoing it
                 return 'block'
         if was_bulked:


@@ -374,7 +382,7 @@ if __name__ == '__main__':
     fd.close()
 
     # TODO write logs (need to collect some statistics)
     fd.close()
 
     # TODO write logs (need to collect some statistics)

-    logstr = datetime.now().strftime('%Y-%m-%d %H:%M') + ' %s %s\n'
+    logstr = datetime.now().strftime('%d/%b/%Y:%H:%M:%S') + ' %s %s\n'

     fd = open(LOGFILE, 'a')
     if unblocked:
         fd.write(logstr % ('unblocked:', ', '.join(unblocked)))
     fd = open(LOGFILE, 'a')
     if unblocked:
         fd.write(logstr % ('unblocked:', ', '.join(unblocked)))


@@ -388,7 +396,7 @@ if __name__ == '__main__':
         fd.write(logstr % (' ua block:', ', '.join(uablocked)))
     if blocked:
         fd.write(logstr % ('new block:', ', '.join(blocked)))
         fd.write(logstr % (' ua block:', ', '.join(uablocked)))
     if blocked:
         fd.write(logstr % ('new block:', ', '.join(blocked)))

-    for k,v in stats.items():
-        if v.ignores_warnings() and k not in notlogged:
-            fd.write(logstr % ('Warning ignored:', k))
+    #for k,v in stats.items():
+    #    if v.ignores_warnings(k in bl.prevblocks) and k not in notlogged and ':' not in k:
+    #        fd.write(logstr % ('Warning ignored:', k))

     fd.close()
     fd.close()