From: Tom Hughes Date: Fri, 16 Jul 2021 14:32:53 +0000 (+0100) Subject: Add IP based rate limiting to the render servers X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/28360d6f39fa7fe8229ed64b4c93d1d0405add47 Add IP based rate limiting to the render servers --- diff --git a/cookbooks/tile/attributes/default.rb b/cookbooks/tile/attributes/default.rb index 2336205d9..cdcc13155 100644 --- a/cookbooks/tile/attributes/default.rb +++ b/cookbooks/tile/attributes/default.rb @@ -13,6 +13,9 @@ default[:tile][:replication][:url] = "https://planet.osm.org/replication/minute/ default[:tile][:data] = {} default[:tile][:styles] = {} +default[:tile][:ratelimit][:requests_per_second] = 20 +default[:tile][:ratelimit][:maximum_backlog] = 3600 + default[:postgresql][:versions] |= [node[:tile][:database][:cluster].split("/").first] default[:accounts][:users][:tile][:status] = :role diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 957caa8ca..1bc45e92f 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -84,6 +84,18 @@ directory "/srv/tile.openstreetmap.org" do mode "755" end +directory "/srv/tile.openstreetmap.org/conf" do + owner "tile" + group "tile" + mode "755" +end + +file "/srv/tile.openstreetmap.org/conf/ip.map" do + owner "tile" + group "adm" + mode "644" +end + package "renderd" systemd_service "renderd" do @@ -482,6 +494,10 @@ package %w[ python3-pyproj ] +gem_package "apachelogregex" +gem_package "file-tail" +gem_package "lru_redux" + remote_directory "/usr/local/bin" do source "bin" owner "root" @@ -492,6 +508,35 @@ remote_directory "/usr/local/bin" do files_mode "755" end +template "/usr/local/bin/tile-ratelimit" do + source "tile-ratelimit.erb" + owner "root" + group "root" + mode "755" +end + +systemd_service "tile-ratelimit" do + description "Monitor tile requests and enforce rate limits" + after "apache2.service" + user "tile" + group "adm" + exec_start "/usr/local/bin/tile-ratelimit" + private_tmp true + private_devices true + private_network true + protect_system "full" + protect_home true + read_write_paths "/srv/tile.openstreetmap.org/conf" + no_new_privileges true + restart "on-failure" +end + +service "tile-ratelimit" do + action [:enable, :start] + subscribes :restart, "file[/usr/local/bin/time-ratelimit]" + subscribes :restart, "systemd_service[tile-ratelimit]" +end + template "/usr/local/bin/expire-tiles" do source "expire-tiles.erb" owner "root" diff --git a/cookbooks/tile/templates/default/apache.erb b/cookbooks/tile/templates/default/apache.erb index a34f2b5d1..f7cba541b 100644 --- a/cookbooks/tile/templates/default/apache.erb +++ b/cookbooks/tile/templates/default/apache.erb @@ -47,6 +47,11 @@ # Enable the rewrite engine RewriteEngine on + # Enforce rate limits + RewriteMap ipmap txt:/srv/tile.openstreetmap.org/conf/ip.map + RewriteCond ${ipmap:%{REMOTE_ADDR}} ^.+$ + RewriteRule ^.*$ /${ipmap:%{REMOTE_ADDR}} [PT] + # Rewrite tile requests to the default style RewriteRule ^/(\d+)/(\d+)/(\d+)\.png$ /default/$1/$2/$3.png [PT,T=image/png,L] RewriteRule ^/(\d+)/(\d+)/(\d+)\.png/status/?$ /default/$1/$2/$3.png/status [PT,T=text/plain,L] @@ -59,6 +64,12 @@ # Redirect ACME certificate challenges RedirectPermanent /.well-known/acme-challenge/ http://acme.openstreetmap.org/.well-known/acme-challenge/ + + # Internal endpoint for blocked users + + Header always set Cache-Control private + Redirect 429 + diff --git a/cookbooks/tile/templates/default/tile-ratelimit.erb b/cookbooks/tile/templates/default/tile-ratelimit.erb new file mode 100755 index 000000000..f0483b4d7 --- /dev/null +++ b/cookbooks/tile/templates/default/tile-ratelimit.erb @@ -0,0 +1,86 @@ +#!/usr/bin/ruby + +require "apache_log_regex" +require "date" +require "file-tail" +require "gdbm" +require "lru_redux" + +REQUESTS_PER_SECOND = <%= node[:tile][:ratelimit][:requests_per_second] %> +BLOCK_AT = <%= node[:tile][:ratelimit][:maximum_backlog] %> +UNBLOCK_AT = BLOCK_AT / 2 + +parser = ApacheLogRegex.new('%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"') +clients = LruRedux::Cache.new(1000000) + +def decay_count(client, time) + decay = (time.to_i - client[:last_update]) * REQUESTS_PER_SECOND + + client[:request_count] = [client[:request_count] - decay, 0].max + client[:last_update] = time.to_i +end + +def write_blocked_ips(clients) + time = Time.now + + File.open("/srv/tile.openstreetmap.org/conf/ip.map.new", "w") do |file| + clients.each do |address, client| + decay_count(client, time) + + if client[:request_count] >= UNBLOCK_AT + file.puts "#{address} blocked" + elsif client.has_key?(:blocked_at) + puts "Unblocked #{address}" + + client.delete(:blocked_at) + end + end + end + + File.rename("/srv/tile.openstreetmap.org/conf/ip.map.new", + "/srv/tile.openstreetmap.org/conf/ip.map") + + time + 900 +end + +next_check = write_blocked_ips(clients) + +File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line| + begin + hash = parser.parse!(line) + + address = hash["%a"] + + next if address == "127.0.0.1" || address == "::1" + + time = Time.now + + client = clients.getset(address) do + { :request_count => 0, :last_update => 0 } + end + + decay_count(client, time) + + client[:request_count] = client[:request_count] + 1 + + if client[:request_count] > BLOCK_AT && !client.has_key?(:blocked_at) + puts "Blocked #{address}" + + client[:blocked_at] = time + + next_check = time + elsif client[:request_count] < UNBLOCK_AT && client.has_key?(:blocked_at) + puts "Unblocked #{address}" + + client.delete(:blocked_at) + + next_check = time + end + + if time >= next_check + next_check = write_blocked_ips(clients) + end + rescue ApacheLogRegex::ParseError + # nil + end +end diff --git a/roles/pyrene.rb b/roles/pyrene.rb index ebc56ae9c..11cc90954 100644 --- a/roles/pyrene.rb +++ b/roles/pyrene.rb @@ -66,6 +66,10 @@ default_attributes( { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 } ] } + }, + :ratelimit => { + :requests_per_second => 40, + :maximum_backlog => 7200 } } )