cookbooks/tile/templates/default/tile-ratelimit.erb

   1 #!/usr/bin/ruby
   2
   3 require "apache_log_regex"
   4 require "date"
   5 require "file-tail"
   6 require "gdbm"
   7 require "lru_redux"
   8
   9 REQUESTS_PER_SECOND = <%= node[:tile][:ratelimit][:requests_per_second] %>
  10 BLOCK_AT = <%= node[:tile][:ratelimit][:maximum_backlog] %>
  11 UNBLOCK_AT = BLOCK_AT / 2
  12
  13 parser = ApacheLogRegex.new('%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"')
  14 clients = LruRedux::Cache.new(1000000)
  15
  16 def decay_count(client, time)
  17   decay = (time.to_i - client[:last_update]) * REQUESTS_PER_SECOND
  18
  19   client[:request_count] = [client[:request_count] - decay, 0].max
  20   client[:last_update] = time.to_i
  21 end
  22
  23 def write_blocked_ips(clients)
  24   time = Time.now
  25
  26   File.open("/srv/tile.openstreetmap.org/conf/ip.map.new", "w") do |file|
  27     clients.each do |address, client|
  28       decay_count(client, time)
  29
  30       if client[:request_count] >= UNBLOCK_AT
  31         file.puts "#{address} blocked"
  32       elsif client.has_key?(:blocked_at)
  33         puts "Unblocked #{address}"
  34
  35         client.delete(:blocked_at)
  36       end
  37     end
  38   end
  39
  40   File.rename("/srv/tile.openstreetmap.org/conf/ip.map.new",
  41               "/srv/tile.openstreetmap.org/conf/ip.map")
  42
  43   time + 900
  44 end
  45
  46 next_check = write_blocked_ips(clients)
  47
  48 File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line|
  49   begin
  50     hash = parser.parse!(line)
  51
  52     address = hash["%a"]
  53     request = hash["%r"]
  54
  55     next if address == "127.0.0.1" || address == "::1"
  56
  57     time = Time.now
  58
  59     client = clients.getset(address) do
  60       { :request_count => 0, :last_update => 0 }
  61     end
  62
  63     decay_count(client, time)
  64
  65     if request =~ %r{^(GET|POST) /cgi-bin/export.*}
  66       client[:request_count] = client[:request_count] + 150
  67     else
  68       client[:request_count] = client[:request_count] + 1
  69     end
  70
  71     if client[:request_count] > BLOCK_AT && !client.has_key?(:blocked_at)
  72       puts "Blocked #{address}"
  73
  74       client[:blocked_at] = time
  75
  76       next_check = time
  77     elsif client[:request_count] < UNBLOCK_AT && client.has_key?(:blocked_at)
  78       puts "Unblocked #{address}"
  79
  80       client.delete(:blocked_at)
  81
  82       next_check = time
  83     end
  84
  85     if time >= next_check
  86       next_check = write_blocked_ips(clients)
  87     end
  88   rescue ApacheLogRegex::ParseError
  89     # nil
  90   end
  91 end