cookbooks/web/templates/default/api-statistics.erb

   1 #!/usr/bin/ruby
   2
   3 require "apache_log_regex"
   4 require "file-tail"
   5 require "json"
   6
   7 def categorise_uri(line)
   8   uri = line.split(" ")[1]
   9
  10   case uri
  11   when %r{api/0\.6/map} then :map
  12   when %r{api/0\.6/changeset/[0-9]*/upload} then :upload
  13   when %r{api/0\.6/amf} then :amf
  14   when %r{api/0\.6/(node|way|relation)/[0-9]*/history} then :history
  15   when %r{api/0\.6/(node|way|relation)/[0-9]*/full} then :full
  16   when %r{api/0\.6/trackpoints} then :trkpts
  17   when %r{api/0\.6/} then :other
  18   else :web
  19   end
  20 end
  21
  22 parser = ApacheLogRegex.new('%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %Dus %{UNIQUE_ID}e %{SSL_PROTOCOL}x %{SSL_CIPHER}x')
  23 last_write = Time.now
  24
  25 statistics = {
  26   :status => Hash.new(0),
  27   :uri => Hash.new(0),
  28   :count => Hash.new(0),
  29   :bytes => Hash.new(0),
  30   :seconds => Hash.new(0.0),
  31   :ssl => Hash.new(0)
  32 }
  33
  34 File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line|
  35   begin
  36     hash = parser.parse(line)
  37
  38     uri = categorise_uri(hash["%r"])
  39     status = hash["%>s"]
  40     bytes = hash["%O"].to_i
  41     seconds = hash["%Dus"].to_f / 1000000
  42     protocol = hash["%{SSL_PROTOCOL}x"]
  43     cipher = hash["%{SSL_CIPHER}x"]
  44
  45     statistics[:status][status] += 1
  46     statistics[:uri][uri] += 1
  47     statistics[:count][[uri, status]] += 1
  48     statistics[:bytes][[uri, status]] += bytes
  49     statistics[:seconds][[uri, status]] += seconds
  50     statistics[:ssl][[protocol, cipher]] += 1 unless protocol == "-"
  51   rescue ApacheLogRegex::ParseError
  52     # nil
  53   end
  54
  55   if Time.now - last_write > 10
  56     File.write("/srv/www.openstreetmap.org/rails/tmp/statistics.json", statistics.slice(:status, :uri).to_json)
  57
  58     File.open("/var/lib/prometheus/node-exporter/api.tmp", "w") do |file|
  59       file.puts "# HELP api_call_count_total Number of calls by type and status"
  60       file.puts "# TYPE api_call_count_total counter"
  61
  62       statistics[:count].each do |key, value|
  63         uri, status = key
  64
  65         file.puts "api_call_count_total{uri=\"#{uri}\",status=\"#{status}\"} #{value}"
  66       end
  67
  68       file.puts "# HELP api_call_bytes_total Number of bytes returned by type and status"
  69       file.puts "# TYPE api_call_bytes_total counter"
  70
  71       statistics[:bytes].each do |key, value|
  72         uri, status = key
  73
  74         file.puts "api_call_bytes_total{uri=\"#{uri}\",status=\"#{status}\"} #{value}"
  75       end
  76
  77       file.puts "# HELP api_call_seconds_total Number of seconds returned by type and status"
  78       file.puts "# TYPE api_call_seconds_total counter"
  79
  80       statistics[:seconds].each do |key, value|
  81         uri, status = key
  82
  83         file.puts "api_call_seconds_total{uri=\"#{uri}\",status=\"#{status}\"} #{value}"
  84       end
  85
  86       file.puts "# HELP api_call_ssl_total Number of calls by SSL protocol and cipher"
  87       file.puts "# TYPE api_call_ssl_total counter"
  88
  89       statistics[:ssl].each do |key, value|
  90         protocol, cipher = key
  91
  92         file.puts "api_call_ssl_total{protocol=\"#{protocol}\",cipher=\"#{cipher}\"} #{value}"
  93       end
  94     end
  95
  96     File.rename("/var/lib/prometheus/node-exporter/api.tmp", "/var/lib/prometheus/node-exporter/api.prom")
  97
  98     last_write = Time.now
  99   end
 100 end