Add an api-statistics daemon to gather data from apache logs

diff --git a/.rubocop.yml b/.rubocop.yml
index 1f5c531bf4865c75bc4c21411ec25c1201afc16c..b6fd6fcc311875c55c8412b766d431fa719dd74a 100644 (file)
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -8,6 +8,7 @@ Style/FileName:
   Exclude:
     - 'cookbooks/trac/files/default/trac-authenticate'
     - 'cookbooks/planet/files/default/replication-bin/replicate-changesets'
+    - 'cookbooks/*/templates/*/*.erb'
     - 'hooks/*'
     - 'roles/*.rb'
 

diff --git a/cookbooks/web/recipes/rails.rb b/cookbooks/web/recipes/rails.rb
index 926ff38ceb12778d7a1c5874a5a841de561adf00..52a49aee8731414a83c6fcb9b01655279b44a49a 100644 (file)
--- a/cookbooks/web/recipes/rails.rb
+++ b/cookbooks/web/recipes/rails.rb
@@ -66,3 +66,27 @@ rails_port "www.openstreetmap.org" do
   oauth_key web_passwords["oauth_key"]
   piwik_configuration piwik_configuration
 end
+
+gem_package "apachelogregex"
+gem_package "file-tail"
+
+template "/usr/local/bin/api-statistics" do
+  source "api-statistics.erb"
+  owner "root"
+  group "root"
+  mode 0755
+end
+
+template "/etc/init.d/api-statistics" do
+  source "api-statistics.init.erb"
+  owner "root"
+  group "root"
+  mode 0755
+end
+
+service "api-statistics" do
+  action [:enable, :start]
+  supports :restart => true
+  subscribes :restart, "template[/usr/local/bin/api-statistics]"
+  subscribes :restart, "template[/etc/init.d/api-statistics]"
+end

diff --git a/cookbooks/web/templates/default/api-statistics.erb b/cookbooks/web/templates/default/api-statistics.erb
new file mode 100644 (file)
index 0000000..af577f3
--- /dev/null
+++ b/cookbooks/web/templates/default/api-statistics.erb
@@ -0,0 +1,42 @@
+#!/usr/bin/ruby
+
+require "apache_log_regex"
+require "file-tail"
+require "json"
+
+def categorise_uri(line)
+  uri = line.split(" ")[1]
+
+  case uri
+  when %r{api/0\.6/map} then :map
+  when %r{api/0\.6/changeset/[0-9]*/upload} then :upload
+  when %r{api/0\.6/amf} then :amf
+  when %r{api/0\.6/(node|way|relation)/[0-9]*/history} then :history
+  when %r{api/0\.6/(node|way|relation)/[0-9]*/full} then :full
+  when %r{api/0\.6/trackpoints} then :trkpts
+  when %r{api/0\.6/} then :other
+  else :web
+  end
+end
+
+parser = ApacheLogRegex.new('%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %Ts')
+last_write = Time.now
+statistics = { :status => Hash.new(0), :uri => Hash.new(0) }
+
+File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line|
+  begin
+    hash = parser.parse(line)
+    status = hash["%>s"]
+    uri = categorise_uri(hash["%r"])
+
+    statistics[:status][status] += 1
+    statistics[:uri][uri] += 1
+  rescue ApacheLogRegex::ParseError
+    # nil
+  end
+
+  if Time.now - last_write > 10
+    File.write("/srv/www.openstreetmap.org/rails/tmp/statistics.json", statistics.to_json)
+    last_write = Time.now
+  end
+end

diff --git a/cookbooks/web/templates/default/api-statistics.init.erb b/cookbooks/web/templates/default/api-statistics.init.erb
new file mode 100644 (file)
index 0000000..880f1da
--- /dev/null
+++ b/cookbooks/web/templates/default/api-statistics.init.erb
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+start() {
+  start-stop-daemon --start --chuid rails:adm --background --make-pidfile --pidfile /var/run/api-statistics.pid --exec /usr/local/bin/api-statistics
+}
+
+stop() {
+  start-stop-daemon --stop --retry 300 --pidfile /var/run/api-statistics.pid
+}
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart)
+    stop || exit $?
+    start
+    ;;
+esac