From be5f5bf83260f6fe16c253bb2069b34bd7421549 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 1 Apr 2015 00:10:34 +0100 Subject: [PATCH] Add an api-statistics daemon to gather data from apache logs --- .rubocop.yml | 1 + cookbooks/web/recipes/rails.rb | 24 +++++++++++ .../web/templates/default/api-statistics.erb | 42 +++++++++++++++++++ .../templates/default/api-statistics.init.erb | 24 +++++++++++ 4 files changed, 91 insertions(+) create mode 100644 cookbooks/web/templates/default/api-statistics.erb create mode 100644 cookbooks/web/templates/default/api-statistics.init.erb diff --git a/.rubocop.yml b/.rubocop.yml index 1f5c531bf..b6fd6fcc3 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -8,6 +8,7 @@ Style/FileName: Exclude: - 'cookbooks/trac/files/default/trac-authenticate' - 'cookbooks/planet/files/default/replication-bin/replicate-changesets' + - 'cookbooks/*/templates/*/*.erb' - 'hooks/*' - 'roles/*.rb' diff --git a/cookbooks/web/recipes/rails.rb b/cookbooks/web/recipes/rails.rb index 926ff38ce..52a49aee8 100644 --- a/cookbooks/web/recipes/rails.rb +++ b/cookbooks/web/recipes/rails.rb @@ -66,3 +66,27 @@ rails_port "www.openstreetmap.org" do oauth_key web_passwords["oauth_key"] piwik_configuration piwik_configuration end + +gem_package "apachelogregex" +gem_package "file-tail" + +template "/usr/local/bin/api-statistics" do + source "api-statistics.erb" + owner "root" + group "root" + mode 0755 +end + +template "/etc/init.d/api-statistics" do + source "api-statistics.init.erb" + owner "root" + group "root" + mode 0755 +end + +service "api-statistics" do + action [:enable, :start] + supports :restart => true + subscribes :restart, "template[/usr/local/bin/api-statistics]" + subscribes :restart, "template[/etc/init.d/api-statistics]" +end diff --git a/cookbooks/web/templates/default/api-statistics.erb b/cookbooks/web/templates/default/api-statistics.erb new file mode 100644 index 000000000..af577f3e9 --- /dev/null +++ b/cookbooks/web/templates/default/api-statistics.erb @@ -0,0 +1,42 @@ +#!/usr/bin/ruby + +require "apache_log_regex" +require "file-tail" +require "json" + +def categorise_uri(line) + uri = line.split(" ")[1] + + case uri + when %r{api/0\.6/map} then :map + when %r{api/0\.6/changeset/[0-9]*/upload} then :upload + when %r{api/0\.6/amf} then :amf + when %r{api/0\.6/(node|way|relation)/[0-9]*/history} then :history + when %r{api/0\.6/(node|way|relation)/[0-9]*/full} then :full + when %r{api/0\.6/trackpoints} then :trkpts + when %r{api/0\.6/} then :other + else :web + end +end + +parser = ApacheLogRegex.new('%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %Ts') +last_write = Time.now +statistics = { :status => Hash.new(0), :uri => Hash.new(0) } + +File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line| + begin + hash = parser.parse(line) + status = hash["%>s"] + uri = categorise_uri(hash["%r"]) + + statistics[:status][status] += 1 + statistics[:uri][uri] += 1 + rescue ApacheLogRegex::ParseError + # nil + end + + if Time.now - last_write > 10 + File.write("/srv/www.openstreetmap.org/rails/tmp/statistics.json", statistics.to_json) + last_write = Time.now + end +end diff --git a/cookbooks/web/templates/default/api-statistics.init.erb b/cookbooks/web/templates/default/api-statistics.init.erb new file mode 100644 index 000000000..880f1dad9 --- /dev/null +++ b/cookbooks/web/templates/default/api-statistics.init.erb @@ -0,0 +1,24 @@ +#!/bin/bash + +# DO NOT EDIT - This file is being maintained by Chef + +start() { + start-stop-daemon --start --chuid rails:adm --background --make-pidfile --pidfile /var/run/api-statistics.pid --exec /usr/local/bin/api-statistics +} + +stop() { + start-stop-daemon --stop --retry 300 --pidfile /var/run/api-statistics.pid +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop || exit $? + start + ;; +esac -- 2.39.5