Add IP based rate limiting to the render servers

diff --git a/cookbooks/tile/attributes/default.rb b/cookbooks/tile/attributes/default.rb
index 2336205d9e95f03ece9c0bb51cba6869ced72be8..cdcc1315520e2ea23daa5d92fd1e7298b37aab4b 100644 (file)
--- a/cookbooks/tile/attributes/default.rb
+++ b/cookbooks/tile/attributes/default.rb
@@ -13,6 +13,9 @@ default[:tile][:replication][:url] = "https://planet.osm.org/replication/minute/
 default[:tile][:data] = {}
 default[:tile][:styles] = {}
 
+default[:tile][:ratelimit][:requests_per_second] = 20
+default[:tile][:ratelimit][:maximum_backlog] = 3600
+
 default[:postgresql][:versions] |= [node[:tile][:database][:cluster].split("/").first]
 
 default[:accounts][:users][:tile][:status] = :role

diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb
index 957caa8ca50dffa2dc15021a1d6cf56ebe46ea4e..1bc45e92f92082b6629ace57305104967120bb01 100644 (file)
--- a/cookbooks/tile/recipes/default.rb
+++ b/cookbooks/tile/recipes/default.rb
@@ -84,6 +84,18 @@ directory "/srv/tile.openstreetmap.org" do
   mode "755"
 end
 
+directory "/srv/tile.openstreetmap.org/conf" do
+  owner "tile"
+  group "tile"
+  mode "755"
+end
+
+file "/srv/tile.openstreetmap.org/conf/ip.map" do
+  owner "tile"
+  group "adm"
+  mode "644"
+end
+
 package "renderd"
 
 systemd_service "renderd" do
@@ -482,6 +494,10 @@ package %w[
   python3-pyproj
 ]
 
+gem_package "apachelogregex"
+gem_package "file-tail"
+gem_package "lru_redux"
+
 remote_directory "/usr/local/bin" do
   source "bin"
   owner "root"
@@ -492,6 +508,35 @@ remote_directory "/usr/local/bin" do
   files_mode "755"
 end
 
+template "/usr/local/bin/tile-ratelimit" do
+  source "tile-ratelimit.erb"
+  owner "root"
+  group "root"
+  mode "755"
+end
+
+systemd_service "tile-ratelimit" do
+  description "Monitor tile requests and enforce rate limits"
+  after "apache2.service"
+  user "tile"
+  group "adm"
+  exec_start "/usr/local/bin/tile-ratelimit"
+  private_tmp true
+  private_devices true
+  private_network true
+  protect_system "full"
+  protect_home true
+  read_write_paths "/srv/tile.openstreetmap.org/conf"
+  no_new_privileges true
+  restart "on-failure"
+end
+
+service "tile-ratelimit" do
+  action [:enable, :start]
+  subscribes :restart, "file[/usr/local/bin/time-ratelimit]"
+  subscribes :restart, "systemd_service[tile-ratelimit]"
+end
+
 template "/usr/local/bin/expire-tiles" do
   source "expire-tiles.erb"
   owner "root"

diff --git a/cookbooks/tile/templates/default/apache.erb b/cookbooks/tile/templates/default/apache.erb
index a34f2b5d14357ff241718ac439808f9a2098aa08..f7cba541b4144106e0b895ca4dc549345737bbc2 100644 (file)
--- a/cookbooks/tile/templates/default/apache.erb
+++ b/cookbooks/tile/templates/default/apache.erb
@@ -47,6 +47,11 @@
   # Enable the rewrite engine
   RewriteEngine on
 
+  # Enforce rate limits
+  RewriteMap ipmap txt:/srv/tile.openstreetmap.org/conf/ip.map
+  RewriteCond ${ipmap:%{REMOTE_ADDR}} ^.+$
+  RewriteRule ^.*$ /${ipmap:%{REMOTE_ADDR}} [PT]
+
   # Rewrite tile requests to the default style
   RewriteRule ^/(\d+)/(\d+)/(\d+)\.png$ /default/$1/$2/$3.png [PT,T=image/png,L]
   RewriteRule ^/(\d+)/(\d+)/(\d+)\.png/status/?$  /default/$1/$2/$3.png/status [PT,T=text/plain,L]
@@ -59,6 +64,12 @@
 
   # Redirect ACME certificate challenges
   RedirectPermanent /.well-known/acme-challenge/ http://acme.openstreetmap.org/.well-known/acme-challenge/
+
+  # Internal endpoint for blocked users
+  <Location /blocked>
+    Header always set Cache-Control private
+    Redirect 429
+  </Location>
 </VirtualHost>
 
 <VirtualHost *:80>

diff --git a/cookbooks/tile/templates/default/tile-ratelimit.erb b/cookbooks/tile/templates/default/tile-ratelimit.erb
new file mode 100755 (executable)
index 0000000..f0483b4
--- /dev/null
+++ b/cookbooks/tile/templates/default/tile-ratelimit.erb
@@ -0,0 +1,86 @@
+#!/usr/bin/ruby
+
+require "apache_log_regex"
+require "date"
+require "file-tail"
+require "gdbm"
+require "lru_redux"
+
+REQUESTS_PER_SECOND = <%= node[:tile][:ratelimit][:requests_per_second] %>
+BLOCK_AT = <%= node[:tile][:ratelimit][:maximum_backlog] %>
+UNBLOCK_AT = BLOCK_AT / 2
+
+parser = ApacheLogRegex.new('%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"')
+clients = LruRedux::Cache.new(1000000)
+
+def decay_count(client, time)
+  decay = (time.to_i - client[:last_update]) * REQUESTS_PER_SECOND
+
+  client[:request_count] = [client[:request_count] - decay, 0].max
+  client[:last_update] = time.to_i
+end
+
+def write_blocked_ips(clients)
+  time = Time.now
+
+  File.open("/srv/tile.openstreetmap.org/conf/ip.map.new", "w") do |file|
+    clients.each do |address, client|
+      decay_count(client, time)
+
+      if client[:request_count] >= UNBLOCK_AT
+        file.puts "#{address} blocked"
+      elsif client.has_key?(:blocked_at)
+        puts "Unblocked #{address}"
+
+        client.delete(:blocked_at)
+      end
+    end
+  end
+
+  File.rename("/srv/tile.openstreetmap.org/conf/ip.map.new",
+              "/srv/tile.openstreetmap.org/conf/ip.map")
+
+  time + 900
+end
+
+next_check = write_blocked_ips(clients)
+
+File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line|
+  begin
+    hash = parser.parse!(line)
+
+    address = hash["%a"]
+
+    next if address == "127.0.0.1" || address == "::1"
+
+    time = Time.now
+
+    client = clients.getset(address) do
+      { :request_count => 0, :last_update => 0 }
+    end
+
+    decay_count(client, time)
+
+    client[:request_count] = client[:request_count] + 1
+
+    if client[:request_count] > BLOCK_AT && !client.has_key?(:blocked_at)
+      puts "Blocked #{address}"
+
+      client[:blocked_at] = time
+
+      next_check = time
+    elsif client[:request_count] < UNBLOCK_AT && client.has_key?(:blocked_at)
+      puts "Unblocked #{address}"
+
+      client.delete(:blocked_at)
+
+      next_check = time
+    end
+
+    if time >= next_check
+      next_check = write_blocked_ips(clients)
+    end
+  rescue ApacheLogRegex::ParseError
+    # nil
+  end
+end

diff --git a/roles/pyrene.rb b/roles/pyrene.rb
index ebc56ae9c6f44fa0e6171663ed825da634709265..11cc90954fe93b6bf4bcec856bced2fa81d209ec 100644 (file)
--- a/roles/pyrene.rb
+++ b/roles/pyrene.rb
@@ -66,6 +66,10 @@ default_attributes(
           { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 }
         ]
       }
+    },
+    :ratelimit => {
+      :requests_per_second => 40,
+      :maximum_backlog => 7200
     }
   }
 )