munin/nominatim_requests_querylog

   1 #!/usr/bin/python3
   2 #
   3 # Plugin to monitor the types of requsts made to the API
   4 #
   5 # Uses the query log.
   6 #
   7 # Parameters:
   8 #
   9 #       config   (required)
  10 #       autoconf (optional - used by munin-config)
  11 #
  12
  13 import re
  14 import os
  15 import sys
  16 from datetime import datetime, timedelta
  17
  18 CONFIG="""graph_title Requests by API call
  19 graph_args --base 1000 -l 0
  20 graph_vlabel requests per minute
  21 graph_category nominatim
  22 z1.label reverse
  23 z1.draw AREA
  24 z1.type GAUGE
  25 z2.label search (successful)
  26 z2.draw STACK
  27 z2.type GAUGE
  28 z3.label search (no result)
  29 z3.draw STACK
  30 z3.type GAUGE
  31 z4.label details
  32 z4.draw STACK
  33 z4.type GAUGE"""
  34
  35 ENTRY_REGEX = re.compile(r'\[[^]]+\] (?P<dur>[0-9.]+) (?P<numres>\d+) (?P<type>[a-z]+) ')
  36 TIME_REGEX = re.compile(r'\[(?P<t_year>\d\d\d\d)-(?P<t_month>\d\d)-(?P<t_day>\d\d) (?P<t_hour>\d\d):(?P<t_min>\d\d):(?P<t_sec>\d\d)[0-9.]*\] ')
  37
  38
  39 class LogFile:
  40     """ A query log file, unpacked. """
  41
  42     def __init__(self, filename):
  43         self.fd = open(filename, encoding='utf-8', errors='replace')
  44         self.len = os.path.getsize(filename)
  45
  46     def __del__(self):
  47         self.fd.close()
  48
  49     def seek_next(self, abstime):
  50         self.fd.seek(abstime)
  51         self.fd.readline()
  52         l = self.fd.readline()
  53         e = TIME_REGEX.match(l)
  54         if e is None:
  55             return None
  56         e = e.groupdict()
  57         return datetime(int(e['t_year']), int(e['t_month']), int(e['t_day']),
  58                              int(e['t_hour']), int(e['t_min']), int(e['t_sec']))
  59
  60     def seek_to_date(self, target):
  61         # start position for binary search
  62         fromseek = 0
  63         fromdate = self.seek_next(0)
  64         if fromdate > target:
  65             return True
  66         # end position for binary search
  67         toseek = -100
  68         while -toseek < self.len:
  69             todate = self.seek_next(self.len + toseek)
  70             if todate is not None:
  71                 break
  72             toseek -= 100
  73         if todate is None or todate < target:
  74             return False
  75         toseek = self.len + toseek
  76
  77
  78         while True:
  79             bps = (toseek - fromseek) / (todate - fromdate).total_seconds()
  80             newseek = fromseek + int((target - fromdate).total_seconds() * bps)
  81             newdate = self.seek_next(newseek)
  82             if newdate is None:
  83                 return False;
  84             error = abs((target - newdate).total_seconds())
  85             if error < 1:
  86                 return True
  87             if newdate > target:
  88                 toseek = newseek
  89                 todate = newdate
  90                 oldfromseek = fromseek
  91                 fromseek = toseek - error * bps
  92                 while True:
  93                     if fromseek <= oldfromseek:
  94                         fromseek = oldfromseek
  95                         fromdate = self.seek_next(fromseek)
  96                         break
  97                     fromdate = self.seek_next(fromseek)
  98                     if fromdate < target:
  99                         break;
 100                     bps *=2
 101                     fromseek -= error * bps
 102             else:
 103                 fromseek = newseek
 104                 fromdate = newdate
 105                 oldtoseek = toseek
 106                 toseek = fromseek + error * bps
 107                 while True:
 108                     if toseek > oldtoseek:
 109                         toseek = oldtoseek
 110                         todate = self.seek_next(toseek)
 111                         break
 112                     todate = self.seek_next(toseek)
 113                     if todate > target:
 114                         break
 115                     bps *=2
 116                     toseek += error * bps
 117             if toseek - fromseek < 500:
 118                 return True
 119
 120
 121     def loglines(self):
 122         for l in self.fd:
 123             e = ENTRY_REGEX.match(l)
 124             if e is not None:
 125                 yield e.groupdict()
 126
 127
 128 if __name__ == '__main__':
 129
 130     if len(sys.argv) > 1 and sys.argv[1] == 'config':
 131         print(CONFIG)
 132         sys.exit(0)
 133
 134     reverse = 0
 135     searchy = 0
 136     searchn = 0
 137     details = 0
 138     if 'NOMINATIM_QUERYLOG' in os.environ:
 139         lf = LogFile(os.environ['NOMINATIM_QUERYLOG'])
 140         if lf.seek_to_date(datetime.now() - timedelta(minutes=5)):
 141             for l in lf.loglines():
 142                 if l['type'] == 'reverse':
 143                     reverse += 1
 144                 elif  l['type'] == 'search':
 145                     if l['numres'] == '0':
 146                         searchn += 1
 147                     else:
 148                         searchy += 1
 149                 else:
 150                     details += 1
 151
 152
 153     print('z1.value', reverse/5)
 154     print('z2.value', searchy/5)
 155     print('z3.value', searchn/5)
 156     print('z4.value', details/5)