From 10d2c8a50a01aecc5fab09b276a96527c368dde8 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 11 Feb 2023 20:24:58 -0800 Subject: [PATCH] Remove tile per IP rate limiting With the ability to ratelimit on the CDN, this is no longer necessary --- cookbooks/tile/attributes/default.rb | 3 - cookbooks/tile/recipes/default.rb | 49 ++-------- cookbooks/tile/templates/default/apache.erb | 11 --- .../tile/templates/default/tile-ratelimit.erb | 91 ------------------- roles/pyrene.rb | 4 - 5 files changed, 9 insertions(+), 149 deletions(-) delete mode 100755 cookbooks/tile/templates/default/tile-ratelimit.erb diff --git a/cookbooks/tile/attributes/default.rb b/cookbooks/tile/attributes/default.rb index a44aa1455..ef3efc3da 100644 --- a/cookbooks/tile/attributes/default.rb +++ b/cookbooks/tile/attributes/default.rb @@ -13,9 +13,6 @@ default[:tile][:replication][:url] = "https://planet.openstreetmap.org/replicati default[:tile][:data] = {} default[:tile][:styles] = {} -default[:tile][:ratelimit][:requests_per_second] = 15 -default[:tile][:ratelimit][:maximum_backlog] = 1800 - default[:postgresql][:versions] |= [node[:tile][:database][:cluster].split("/").first] default[:accounts][:users][:tile][:status] = :role diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 979c012d5..6765252a7 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -96,16 +96,10 @@ directory "/srv/tile.openstreetmap.org" do mode "755" end +# Old directory for IP rate limiting, now on the CDN directory "/srv/tile.openstreetmap.org/conf" do - owner "tile" - group "tile" - mode "755" -end - -file "/srv/tile.openstreetmap.org/conf/ip.map" do - owner "tile" - group "adm" - mode "644" + action :delete + recursive true end tile_directories = node[:tile][:styles].collect do |_, style| @@ -524,18 +518,6 @@ package %w[ python3-pyproj ] -gem_package "apachelogregex" do - gem_binary node[:ruby][:gem] -end - -gem_package "file-tail" do - gem_binary node[:ruby][:gem] -end - -gem_package "lru_redux" do - gem_binary node[:ruby][:gem] -end - remote_directory "/usr/local/bin" do source "bin" owner "root" @@ -546,29 +528,16 @@ remote_directory "/usr/local/bin" do files_mode "755" end -template "/usr/local/bin/tile-ratelimit" do - source "tile-ratelimit.erb" - owner "root" - group "root" - mode "755" +file "/usr/local/bin/tile-ratelimit" do + action :delete end -systemd_service "tile-ratelimit" do - description "Monitor tile requests and enforce rate limits" - after "apache2.service" - user "tile" - group "adm" - exec_start "/usr/local/bin/tile-ratelimit" - nice 10 - sandbox true - read_write_paths "/srv/tile.openstreetmap.org/conf" - restart "on-failure" +service "tile-ratelimit" do + action [:stop, :disable] end -service "tile-ratelimit" do - action [:enable, :start] - subscribes :restart, "file[/usr/local/bin/tile-ratelimit]" - subscribes :restart, "systemd_service[tile-ratelimit]" +systemd_service "tile-ratelimit" do + action :delete end template "/usr/local/bin/expire-tiles" do diff --git a/cookbooks/tile/templates/default/apache.erb b/cookbooks/tile/templates/default/apache.erb index c24e06e82..7884ee1f4 100644 --- a/cookbooks/tile/templates/default/apache.erb +++ b/cookbooks/tile/templates/default/apache.erb @@ -47,11 +47,6 @@ # Enable the rewrite engine RewriteEngine on - # Enforce rate limits - RewriteMap ipmap txt:/srv/tile.openstreetmap.org/conf/ip.map - RewriteCond ${ipmap:%{REMOTE_ADDR}} ^.+$ - RewriteRule ^.*$ /${ipmap:%{REMOTE_ADDR}} [PT] - # Rewrite tile requests to the default style RewriteRule ^/(\d+)/(\d+)/(\d+)\.png$ /default/$1/$2/$3.png [PT,T=image/png,L] RewriteRule ^/(\d+)/(\d+)/(\d+)\.png/status/?$ /default/$1/$2/$3.png/status [PT,T=text/plain,L] @@ -91,12 +86,6 @@ # OSM UCL IPv4 Require ip 193.60.236.0/24 - - # Internal endpoint for blocked users - - Header always set Cache-Control private - Redirect 429 - diff --git a/cookbooks/tile/templates/default/tile-ratelimit.erb b/cookbooks/tile/templates/default/tile-ratelimit.erb deleted file mode 100755 index 63e7711b3..000000000 --- a/cookbooks/tile/templates/default/tile-ratelimit.erb +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/ruby - -require "apache_log_regex" -require "date" -require "file-tail" -require "gdbm" -require "lru_redux" - -REQUESTS_PER_SECOND = <%= node[:tile][:ratelimit][:requests_per_second] %> -BLOCK_AT = <%= node[:tile][:ratelimit][:maximum_backlog] %> -UNBLOCK_AT = BLOCK_AT / 2 - -parser = ApacheLogRegex.new('%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"') -clients = LruRedux::Cache.new(1000000) - -def decay_count(client, time) - decay = (time.to_i - client[:last_update]) * REQUESTS_PER_SECOND - - client[:request_count] = [client[:request_count] - decay, 0].max - client[:last_update] = time.to_i -end - -def write_blocked_ips(clients) - time = Time.now - - File.open("/srv/tile.openstreetmap.org/conf/ip.map.new", "w") do |file| - clients.each do |address, client| - decay_count(client, time) - - if client[:request_count] >= UNBLOCK_AT - file.puts "#{address} blocked" - elsif client.has_key?(:blocked_at) - puts "Unblocked #{address}" - - client.delete(:blocked_at) - end - end - end - - File.rename("/srv/tile.openstreetmap.org/conf/ip.map.new", - "/srv/tile.openstreetmap.org/conf/ip.map") - - time + 900 -end - -next_check = write_blocked_ips(clients) - -File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line| - begin - hash = parser.parse!(line) - - address = hash["%a"] - request = hash["%r"] - - next if address == "127.0.0.1" || address == "::1" - - time = Time.now - - client = clients.getset(address) do - { :request_count => 0, :last_update => 0 } - end - - decay_count(client, time) - - if request =~ %r{^(GET|POST) /cgi-bin/export.*} - client[:request_count] = client[:request_count] + 150 - else - client[:request_count] = client[:request_count] + 1 - end - - if client[:request_count] > BLOCK_AT && !client.has_key?(:blocked_at) - puts "Blocked #{address}" - - client[:blocked_at] = time - - next_check = time - elsif client[:request_count] < UNBLOCK_AT && client.has_key?(:blocked_at) - puts "Unblocked #{address}" - - client.delete(:blocked_at) - - next_check = time - end - - if time >= next_check - next_check = write_blocked_ips(clients) - end - rescue ApacheLogRegex::ParseError - # nil - end -end diff --git a/roles/pyrene.rb b/roles/pyrene.rb index ff1b2fc4b..a004062b3 100644 --- a/roles/pyrene.rb +++ b/roles/pyrene.rb @@ -67,10 +67,6 @@ default_attributes( ] } }, - :ratelimit => { - :requests_per_second => 30, - :maximum_backlog => 3600 - } } ) -- 2.39.5