cookbooks/web/templates/default/api-statistics.erb

   1 #!/usr/bin/ruby
   2
   3 require "apache_log_regex"
   4 require "file-tail"
   5 require "json"
   6
   7 def categorise_uri(line)
   8   uri = line.split(" ")[1]
   9
  10   case uri
  11   when %r{api/0\.6/map} then :map
  12   when %r{api/0\.6/changeset/[0-9]*/upload} then :upload
  13   when %r{api/0\.6/amf} then :amf
  14   when %r{api/0\.6/(node|way|relation)/[0-9]*/history} then :history
  15   when %r{api/0\.6/(node|way|relation)/[0-9]*/full} then :full
  16   when %r{api/0\.6/trackpoints} then :trkpts
  17   when %r{api/0\.6/} then :other
  18   else :web
  19   end
  20 end
  21
  22 parser = ApacheLogRegex.new('%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %Dus %{UNIQUE_ID}e %{SSL_PROTOCOL}x %{SSL_CIPHER}x %{AUTH_METHOD}e')
  23 last_write = Time.now
  24
  25 statistics = {
  26   :status => Hash.new(0),
  27   :uri => Hash.new(0),
  28   :count => Hash.new(0),
  29   :bytes => Hash.new(0),
  30   :seconds => Hash.new(0.0),
  31   :ssl => Hash.new(0),
  32   :auth => Hash.new(0)
  33 }
  34
  35 File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line|
  36   begin
  37     hash = parser.parse!(line)
  38
  39     uri = categorise_uri(hash["%r"])
  40     status = hash["%>s"]
  41     bytes = hash["%O"].to_i
  42     seconds = hash["%Dus"].to_f / 1000000
  43     protocol = hash["%{SSL_PROTOCOL}x"]
  44     cipher = hash["%{SSL_CIPHER}x"]
  45     auth = hash["%{AUTH_METHOD}e"]
  46
  47     statistics[:status][status] += 1
  48     statistics[:uri][uri] += 1
  49     statistics[:count][[uri, status]] += 1
  50     statistics[:bytes][[uri, status]] += bytes
  51     statistics[:seconds][[uri, status]] += seconds
  52     statistics[:ssl][[protocol, cipher]] += 1 unless protocol == "-"
  53     statistics[:auth][auth] += 1 unless auth == "-"
  54   rescue ApacheLogRegex::ParseError => ex
  55     STDERR.puts ex.to_s
  56   end
  57
  58   if Time.now - last_write > 10
  59     File.write("/srv/www.openstreetmap.org/rails/tmp/statistics.json", statistics.slice(:status, :uri).to_json)
  60
  61     File.open("/var/lib/prometheus/node-exporter/api.tmp", "w") do |file|
  62       file.puts "# HELP api_call_count_total Number of calls by type and status"
  63       file.puts "# TYPE api_call_count_total counter"
  64
  65       statistics[:count].each do |key, value|
  66         uri, status = key
  67
  68         file.puts "api_call_count_total{uri=\"#{uri}\",status=\"#{status}\"} #{value}"
  69       end
  70
  71       file.puts "# HELP api_call_bytes_total Number of bytes returned by type and status"
  72       file.puts "# TYPE api_call_bytes_total counter"
  73
  74       statistics[:bytes].each do |key, value|
  75         uri, status = key
  76
  77         file.puts "api_call_bytes_total{uri=\"#{uri}\",status=\"#{status}\"} #{value}"
  78       end
  79
  80       file.puts "# HELP api_call_seconds_total Number of seconds returned by type and status"
  81       file.puts "# TYPE api_call_seconds_total counter"
  82
  83       statistics[:seconds].each do |key, value|
  84         uri, status = key
  85
  86         file.puts "api_call_seconds_total{uri=\"#{uri}\",status=\"#{status}\"} #{value}"
  87       end
  88
  89       file.puts "# HELP api_call_ssl_total Number of calls by SSL protocol and cipher"
  90       file.puts "# TYPE api_call_ssl_total counter"
  91
  92       statistics[:ssl].each do |key, value|
  93         protocol, cipher = key
  94
  95         file.puts "api_call_ssl_total{protocol=\"#{protocol}\",cipher=\"#{cipher}\"} #{value}"
  96       end
  97
  98       file.puts "# HELP api_call_auth_method_total Number of calls by authentication method"
  99       file.puts "# TYPE api_call_auth_method_total counter"
 100
 101       statistics[:auth].each do |method, value|
 102         file.puts "api_call_auth_method_total{method=\"#{method}\"} #{value}"
 103       end
 104     end
 105
 106     File.rename("/var/lib/prometheus/node-exporter/api.tmp", "/var/lib/prometheus/node-exporter/api.prom")
 107
 108     last_write = Time.now
 109   end
 110 end