Remove tile per IP rate limiting

With the ability to ratelimit on the CDN, this is no longer necessary

diff --git a/cookbooks/tile/attributes/default.rb b/cookbooks/tile/attributes/default.rb
index a44aa145550091806449ec46a47549446912f583..ef3efc3da49ad9ffb5379eb17ad567b0c01f3c63 100644 (file)
--- a/cookbooks/tile/attributes/default.rb
+++ b/cookbooks/tile/attributes/default.rb
@@ -13,9 +13,6 @@ default[:tile][:replication][:url] = "https://planet.openstreetmap.org/replicati
 default[:tile][:data] = {}
 default[:tile][:styles] = {}
 
-default[:tile][:ratelimit][:requests_per_second] = 15
-default[:tile][:ratelimit][:maximum_backlog] = 1800
-
 default[:postgresql][:versions] |= [node[:tile][:database][:cluster].split("/").first]
 
 default[:accounts][:users][:tile][:status] = :role

diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb
index 979c012d536de878485727d077af7cbf3cd66c2c..6765252a7b9c77b72050c35e33b822b8898a198f 100644 (file)
--- a/cookbooks/tile/recipes/default.rb
+++ b/cookbooks/tile/recipes/default.rb
@@ -96,16 +96,10 @@ directory "/srv/tile.openstreetmap.org" do
   mode "755"
 end
 
+# Old directory for IP rate limiting, now on the CDN
 directory "/srv/tile.openstreetmap.org/conf" do
-  owner "tile"
-  group "tile"
-  mode "755"
-end
-
-file "/srv/tile.openstreetmap.org/conf/ip.map" do
-  owner "tile"
-  group "adm"
-  mode "644"
+  action :delete
+  recursive true
 end
 
 tile_directories = node[:tile][:styles].collect do |_, style|
@@ -524,18 +518,6 @@ package %w[
   python3-pyproj
 ]
 
-gem_package "apachelogregex" do
-  gem_binary node[:ruby][:gem]
-end
-
-gem_package "file-tail" do
-  gem_binary node[:ruby][:gem]
-end
-
-gem_package "lru_redux" do
-  gem_binary node[:ruby][:gem]
-end
-
 remote_directory "/usr/local/bin" do
   source "bin"
   owner "root"
@@ -546,29 +528,16 @@ remote_directory "/usr/local/bin" do
   files_mode "755"
 end
 
-template "/usr/local/bin/tile-ratelimit" do
-  source "tile-ratelimit.erb"
-  owner "root"
-  group "root"
-  mode "755"
+file "/usr/local/bin/tile-ratelimit" do
+  action :delete
 end
 
-systemd_service "tile-ratelimit" do
-  description "Monitor tile requests and enforce rate limits"
-  after "apache2.service"
-  user "tile"
-  group "adm"
-  exec_start "/usr/local/bin/tile-ratelimit"
-  nice 10
-  sandbox true
-  read_write_paths "/srv/tile.openstreetmap.org/conf"
-  restart "on-failure"
+service "tile-ratelimit" do
+  action [:stop, :disable]
 end
 
-service "tile-ratelimit" do
-  action [:enable, :start]
-  subscribes :restart, "file[/usr/local/bin/tile-ratelimit]"
-  subscribes :restart, "systemd_service[tile-ratelimit]"
+systemd_service "tile-ratelimit" do
+  action :delete
 end
 
 template "/usr/local/bin/expire-tiles" do

diff --git a/cookbooks/tile/templates/default/apache.erb b/cookbooks/tile/templates/default/apache.erb
index c24e06e821349eb182d9c8e94693aaa099a960a1..7884ee1f4f97d8dfd8d5c4209e80b75e7a23cb97 100644 (file)
--- a/cookbooks/tile/templates/default/apache.erb
+++ b/cookbooks/tile/templates/default/apache.erb
@@ -47,11 +47,6 @@
   # Enable the rewrite engine
   RewriteEngine on
 
-  # Enforce rate limits
-  RewriteMap ipmap txt:/srv/tile.openstreetmap.org/conf/ip.map
-  RewriteCond ${ipmap:%{REMOTE_ADDR}} ^.+$
-  RewriteRule ^.*$ /${ipmap:%{REMOTE_ADDR}} [PT]
-
   # Rewrite tile requests to the default style
   RewriteRule ^/(\d+)/(\d+)/(\d+)\.png$ /default/$1/$2/$3.png [PT,T=image/png,L]
   RewriteRule ^/(\d+)/(\d+)/(\d+)\.png/status/?$  /default/$1/$2/$3.png/status [PT,T=text/plain,L]
@@ -91,12 +86,6 @@
     # OSM UCL IPv4
     Require ip 193.60.236.0/24
   </LocationMatch>
-
-  # Internal endpoint for blocked users
-  <Location /blocked>
-    Header always set Cache-Control private
-    Redirect 429
-  </Location>
 </VirtualHost>
 
 <VirtualHost *:80>

diff --git a/cookbooks/tile/templates/default/tile-ratelimit.erb b/cookbooks/tile/templates/default/tile-ratelimit.erb
deleted file mode 100755 (executable)
index 63e7711..0000000
--- a/cookbooks/tile/templates/default/tile-ratelimit.erb
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/ruby
-
-require "apache_log_regex"
-require "date"
-require "file-tail"
-require "gdbm"
-require "lru_redux"
-
-REQUESTS_PER_SECOND = <%= node[:tile][:ratelimit][:requests_per_second] %>
-BLOCK_AT = <%= node[:tile][:ratelimit][:maximum_backlog] %>
-UNBLOCK_AT = BLOCK_AT / 2
-
-parser = ApacheLogRegex.new('%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"')
-clients = LruRedux::Cache.new(1000000)
-
-def decay_count(client, time)
-  decay = (time.to_i - client[:last_update]) * REQUESTS_PER_SECOND
-
-  client[:request_count] = [client[:request_count] - decay, 0].max
-  client[:last_update] = time.to_i
-end
-
-def write_blocked_ips(clients)
-  time = Time.now
-
-  File.open("/srv/tile.openstreetmap.org/conf/ip.map.new", "w") do |file|
-    clients.each do |address, client|
-      decay_count(client, time)
-
-      if client[:request_count] >= UNBLOCK_AT
-        file.puts "#{address} blocked"
-      elsif client.has_key?(:blocked_at)
-        puts "Unblocked #{address}"
-
-        client.delete(:blocked_at)
-      end
-    end
-  end
-
-  File.rename("/srv/tile.openstreetmap.org/conf/ip.map.new",
-              "/srv/tile.openstreetmap.org/conf/ip.map")
-
-  time + 900
-end
-
-next_check = write_blocked_ips(clients)
-
-File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line|
-  begin
-    hash = parser.parse!(line)
-
-    address = hash["%a"]
-    request = hash["%r"]
-
-    next if address == "127.0.0.1" || address == "::1"
-
-    time = Time.now
-
-    client = clients.getset(address) do
-      { :request_count => 0, :last_update => 0 }
-    end
-
-    decay_count(client, time)
-
-    if request =~ %r{^(GET|POST) /cgi-bin/export.*}
-      client[:request_count] = client[:request_count] + 150
-    else
-      client[:request_count] = client[:request_count] + 1
-    end
-
-    if client[:request_count] > BLOCK_AT && !client.has_key?(:blocked_at)
-      puts "Blocked #{address}"
-
-      client[:blocked_at] = time
-
-      next_check = time
-    elsif client[:request_count] < UNBLOCK_AT && client.has_key?(:blocked_at)
-      puts "Unblocked #{address}"
-
-      client.delete(:blocked_at)
-
-      next_check = time
-    end
-
-    if time >= next_check
-      next_check = write_blocked_ips(clients)
-    end
-  rescue ApacheLogRegex::ParseError
-    # nil
-  end
-end

diff --git a/roles/pyrene.rb b/roles/pyrene.rb
index ff1b2fc4b7519542f53064652097ee3026db9fc0..a004062b375caa335cd832e086844047af3c0482 100644 (file)
--- a/roles/pyrene.rb
+++ b/roles/pyrene.rb
@@ -67,10 +67,6 @@ default_attributes(
         ]
       }
     },
-    :ratelimit => {
-      :requests_per_second => 30,
-      :maximum_backlog => 3600
-    }
   }
 )