munin/nominatim_query_speed_querylog

   1 #!/usr/bin/python3
   2 #
   3 # Plugin to monitor the types of requsts made to the API
   4 #
   5 # Uses the query log.
   6 #
   7 # Parameters:
   8 #
   9 #       config   (required)
  10 #       autoconf (optional - used by munin-config)
  11 #
  12
  13 import re
  14 import os
  15 import sys
  16 from datetime import datetime, timedelta
  17
  18 CONFIG="""graph_title Total Nominatim response time
  19 graph_vlabel Time to response
  20 graph_category Nominatim
  21 graph_period minute
  22 graph_args --base 1000
  23
  24 avgs.label Average search time
  25 avgs.draw LINE
  26 avgs.type GAUGE
  27 avgs.min 0
  28 avgs.info Moving 5 minute average time to perform search
  29
  30 avgr.label Average reverse time
  31 avgr.draw LINE
  32 avgr.type GAUGE
  33 avgr.min 0
  34 avgr.info Moving 5 minute average time to perform reverse
  35
  36 max.label Slowest time to response (1/100)
  37 max.draw LINE
  38 max.type GAUGE
  39 max.min 0
  40 max.info Slowest query in last 5 minutes (unit: 100s)"""
  41
  42 ENTRY_REGEX = re.compile(r'\[[^]]+\] (?P<dur>[0-9.]+) (?P<numres>\d+) (?P<type>[a-z]+) ')
  43 TIME_REGEX = re.compile(r'\[(?P<t_year>\d\d\d\d)-(?P<t_month>\d\d)-(?P<t_day>\d\d) (?P<t_hour>\d\d):(?P<t_min>\d\d):(?P<t_sec>\d\d)[0-9.]*\] ')
  44
  45
  46 class LogFile:
  47     """ A query log file, unpacked. """
  48
  49     def __init__(self, filename):
  50         self.fd = open(filename, encoding='utf-8', errors='replace')
  51         self.len = os.path.getsize(filename)
  52
  53     def __del__(self):
  54         self.fd.close()
  55
  56     def seek_next(self, abstime):
  57         self.fd.seek(abstime)
  58         self.fd.readline()
  59         l = self.fd.readline()
  60         e = TIME_REGEX.match(l)
  61         if e is None:
  62             return None
  63         e = e.groupdict()
  64         return datetime(int(e['t_year']), int(e['t_month']), int(e['t_day']),
  65                              int(e['t_hour']), int(e['t_min']), int(e['t_sec']))
  66
  67     def seek_to_date(self, target):
  68         # start position for binary search
  69         fromseek = 0
  70         fromdate = self.seek_next(0)
  71         if fromdate > target:
  72             return True
  73         # end position for binary search
  74         toseek = -100
  75         while -toseek < self.len:
  76             todate = self.seek_next(self.len + toseek)
  77             if todate is not None:
  78                 break
  79             toseek -= 100
  80         if todate is None or todate < target:
  81             return False
  82         toseek = self.len + toseek
  83
  84
  85         while True:
  86             bps = (toseek - fromseek) / (todate - fromdate).total_seconds()
  87             newseek = fromseek + int((target - fromdate).total_seconds() * bps)
  88             newdate = self.seek_next(newseek)
  89             if newdate is None:
  90                 return False;
  91             error = abs((target - newdate).total_seconds())
  92             if error < 1:
  93                 return True
  94             if newdate > target:
  95                 toseek = newseek
  96                 todate = newdate
  97                 oldfromseek = fromseek
  98                 fromseek = toseek - error * bps
  99                 while True:
 100                     if fromseek <= oldfromseek:
 101                         fromseek = oldfromseek
 102                         fromdate = self.seek_next(fromseek)
 103                         break
 104                     fromdate = self.seek_next(fromseek)
 105                     if fromdate < target:
 106                         break;
 107                     bps *=2
 108                     fromseek -= error * bps
 109             else:
 110                 fromseek = newseek
 111                 fromdate = newdate
 112                 oldtoseek = toseek
 113                 toseek = fromseek + error * bps
 114                 while True:
 115                     if toseek > oldtoseek:
 116                         toseek = oldtoseek
 117                         todate = self.seek_next(toseek)
 118                         break
 119                     todate = self.seek_next(toseek)
 120                     if todate > target:
 121                         break
 122                     bps *=2
 123                     toseek += error * bps
 124             if toseek - fromseek < 500:
 125                 return True
 126
 127
 128     def loglines(self):
 129         for l in self.fd:
 130             e = ENTRY_REGEX.match(l)
 131             if e is not None:
 132                 yield e.groupdict()
 133
 134
 135 if __name__ == '__main__':
 136
 137     if len(sys.argv) > 1 and sys.argv[1] == 'config':
 138         print(CONFIG)
 139         sys.exit(0)
 140
 141     sumrev = 0
 142     numrev = 0
 143     sumsearch = 0
 144     numsearch = 0
 145     maxres = 0
 146     if 'NOMINATIM_QUERYLOG' in os.environ:
 147         lf = LogFile(os.environ['NOMINATIM_QUERYLOG'])
 148         if lf.seek_to_date(datetime.now() - timedelta(minutes=5)):
 149             for l in lf.loglines():
 150                 dur = float(l['dur'])
 151                 if l['type'] == 'reverse':
 152                     numrev += 1
 153                     sumrev += dur
 154                 elif  l['type'] == 'search':
 155                     numsearch += 1
 156                     sumsearch += dur
 157                 if dur > maxres:
 158                     maxres = dur
 159
 160
 161     print('avgs.value', 0 if numsearch == 0 else sumsearch/numsearch)
 162     print('avgr.value', 0 if numrev == 0 else sumrev/numrev)
 163     print('max.value', maxres/100.0)