munin/nominatim_requests_querylog

   1 #!/usr/bin/python3
   2 #
   3 # Plugin to monitor the types of requsts made to the API
   4 #
   5 # Uses the query log.
   6 #
   7 # Parameters:
   8 #
   9 #       config   (required)
  10 #       autoconf (optional - used by munin-config)
  11 #
  12
  13 import re
  14 import os
  15 import sys
  16 from datetime import datetime, timedelta
  17
  18 CONFIG="""graph_title Requests by API call
  19 graph_args --base 1000 -l 0
  20 graph_vlabel requests per minute
  21 graph_category nominatim
  22 z1.label reverse
  23 z1.draw AREA
  24 z1.type GAUGE
  25 z2.label search (successful)
  26 z2.draw STACK
  27 z2.type GAUGE
  28 z3.label search (no result)
  29 z3.draw STACK
  30 z3.type GAUGE
  31 z4.label lookup
  32 z4.draw STACK
  33 z4.type GAUGE
  34 z4.label details
  35 z4.draw STACK
  36 z4.type GAUGE"""
  37
  38 ENTRY_REGEX = re.compile(r'\[[^]]+\] (?P<dur>[0-9.]+) (?P<numres>\d+) (?P<type>[a-z]+) ')
  39 TIME_REGEX = re.compile(r'\[(?P<t_year>\d\d\d\d)-(?P<t_month>\d\d)-(?P<t_day>\d\d) (?P<t_hour>\d\d):(?P<t_min>\d\d):(?P<t_sec>\d\d)[0-9.]*\] ')
  40
  41
  42 class LogFile:
  43     """ A query log file, unpacked. """
  44
  45     def __init__(self, filename):
  46         self.fd = open(filename, encoding='utf-8', errors='replace')
  47         self.len = os.path.getsize(filename)
  48
  49     def __del__(self):
  50         self.fd.close()
  51
  52     def seek_next(self, abstime):
  53         self.fd.seek(abstime)
  54         self.fd.readline()
  55         l = self.fd.readline()
  56         e = TIME_REGEX.match(l)
  57         if e is None:
  58             return None
  59         e = e.groupdict()
  60         return datetime(int(e['t_year']), int(e['t_month']), int(e['t_day']),
  61                              int(e['t_hour']), int(e['t_min']), int(e['t_sec']))
  62
  63     def seek_to_date(self, target):
  64         # start position for binary search
  65         fromseek = 0
  66         fromdate = self.seek_next(0)
  67         if fromdate > target:
  68             return True
  69         # end position for binary search
  70         toseek = -100
  71         while -toseek < self.len:
  72             todate = self.seek_next(self.len + toseek)
  73             if todate is not None:
  74                 break
  75             toseek -= 100
  76         if todate is None or todate < target:
  77             return False
  78         toseek = self.len + toseek
  79
  80
  81         while True:
  82             bps = (toseek - fromseek) / (todate - fromdate).total_seconds()
  83             newseek = fromseek + int((target - fromdate).total_seconds() * bps)
  84             newdate = self.seek_next(newseek)
  85             if newdate is None:
  86                 return False;
  87             error = abs((target - newdate).total_seconds())
  88             if error < 1:
  89                 return True
  90             if newdate > target:
  91                 toseek = newseek
  92                 todate = newdate
  93                 oldfromseek = fromseek
  94                 fromseek = toseek - error * bps
  95                 while True:
  96                     if fromseek <= oldfromseek:
  97                         fromseek = oldfromseek
  98                         fromdate = self.seek_next(fromseek)
  99                         break
 100                     fromdate = self.seek_next(fromseek)
 101                     if fromdate < target:
 102                         break;
 103                     bps *=2
 104                     fromseek -= error * bps
 105             else:
 106                 fromseek = newseek
 107                 fromdate = newdate
 108                 oldtoseek = toseek
 109                 toseek = fromseek + error * bps
 110                 while True:
 111                     if toseek > oldtoseek:
 112                         toseek = oldtoseek
 113                         todate = self.seek_next(toseek)
 114                         break
 115                     todate = self.seek_next(toseek)
 116                     if todate > target:
 117                         break
 118                     bps *=2
 119                     toseek += error * bps
 120             if toseek - fromseek < 500:
 121                 return True
 122
 123
 124     def loglines(self):
 125         for l in self.fd:
 126             e = ENTRY_REGEX.match(l)
 127             if e is not None:
 128                 yield e.groupdict()
 129
 130
 131 if __name__ == '__main__':
 132
 133     if len(sys.argv) > 1 and sys.argv[1] == 'config':
 134         print(CONFIG)
 135         sys.exit(0)
 136
 137     reverse = 0
 138     searchy = 0
 139     searchn = 0
 140     lookup = 0
 141     details = 0
 142     if 'NOMINATIM_QUERYLOG' in os.environ:
 143         lf = LogFile(os.environ['NOMINATIM_QUERYLOG'])
 144         if lf.seek_to_date(datetime.now() - timedelta(minutes=5)):
 145             for l in lf.loglines():
 146                 if l['type'] == 'reverse':
 147                     reverse += 1
 148                 elif  l['type'] == 'search':
 149                     if l['numres'] == '0':
 150                         searchn += 1
 151                     else:
 152                         searchy += 1
 153                 elif  l['type'] == 'place':
 154                     lookup +=1
 155                 else:
 156                     details += 1
 157
 158
 159     print('z1.value', reverse/5)
 160     print('z2.value', searchy/5)
 161     print('z3.value', searchn/5)
 162     print('z4.value', lookup/5)
 163     print('z4.value', details/5)