X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/f24262ba0e7b60e922c8e2c56a8b77437fee8de2..539dbe9f74466a7ec90ea7cbfb9bc8b49911551d:/cookbooks/tilecache/templates/default/nginx_tile.conf.erb diff --git a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb index ce2b23b9a..1965de764 100644 --- a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb +++ b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb @@ -1,77 +1,401 @@ # DO NOT EDIT - This file is being maintained by Chef upstream tile_cache_backend { - server 127.0.0.1:8080; - <% @caches.each do |cache| -%> - <% if cache[:hostname] != node[:hostname] -%> - #Server <%= cache[:hostname] %> - <% cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%> - server <%= address %> backup; - <% end -%> - <% end -%> - <% end -%> - - keepalive 32; + server 127.0.0.1:8080 weight=1000 max_fails=32; + server 127.0.0.2:8080 weight=1000 max_fails=32; + + # Add the tile_siblings caches to relieve pressure if local squid failing + # Balancer: round-robin +<% server_weight = 1000 -%> +<% Array(@node[:tilecache][:tile_siblings]).each do |cache_peer| -%> +<% @caches.each do |cache| -%> +<% if cache_peer == cache[:fqdn] -%> +<% if cache[:hostname] != node[:hostname] -%> +<% cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%> + server <%= address %>:80 weight=<%= server_weight %> max_fails=32 backup; # Server <%= cache[:hostname] %> +<% server_weight -= server_weight.div(2) -%> +<% end -%> +<% end -%> +<% end -%> +<% end -%> +<% end -%> + + keepalive 128; } -# Rates table based on current cookie value -map $cookie_qos_token $limit_rate_qos { - include /etc/nginx/conf.d/tile_qos_rates.map; +# Geo Map of tile caches +geo $tile_cache { + default "client"; +<% @caches.each do |cache| -%> +<% cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%> + <%= address %> "cache"; # <%= cache[:hostname] %> +<% end -%> +<% end -%> } +# Rates table based on current cookie value +# map $cookie__osm_totp_token $limit_rate_qos { +# include /etc/nginx/conf.d/tile_qos_rates.map; +# } + # Set-Cookie table based on current cookie value -map $cookie_qos_token $cookie_qos_token_set { - include /etc/nginx/conf.d/tile_qos_cookies.map; -} +# map $cookie__osm_totp_token $cookie_qos_token_set { +# include /etc/nginx/conf.d/tile_qos_cookies.map; +# } map $http_user_agent $approved_scraper { - default ''; # Not approved - '~^JOSM\/' 'JOSM'; - '~^Mozilla\/5\.0\ QGIS\/' 'QGIS'; + default 0; # Not approved + '~^JOSM\/' 1; # JOSM + '~^Mozilla\/5\.0\ QGIS\/' 1; # QGIS +} + +map $http_user_agent $denied_scraper { + default 0; # Not denied + '' 1; # No User-Agent Set + '-' 1; + + # Library defaults + '~^Python\-urllib\/' 1; + '~^python\-requests\/' 1; + '~^node\-fetch\/' 1; + '~^R$' 1; + '~^Java\/' 1; + '~^tiles$' 1; + '~^okhttp\/' 1; + '~^Microsoft-ATL-Native\/' 1; + '/n software IPWorks HTTP/S Component - www.nsoftware.com' 1; + '~^Wget\/' 1; + 'java' 1; + + # Library defaults or fakes + 'Android' 1; + 'kc_android' 1; + 'host' 1; + '~^maptestapp' 1; + 'Other' 1; + 'osmdroid' 1; + '~^tilelive-http' 1; + '~^Java-http-client' 1; + + # Fakes + 'Mozilla/4.0' 1; + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)' 1; + + # Bulk downloaders + 'C# TilesDownloader' 1; + 'MapDownloader' 1; + '~^staticmaps' 1; + + # Overusage apps + '~^runtastic' 1; + '~^Where\ my\ children' 1; + 'nossoonibusjp.android.crosswalk' 1; + 'br.com.concisoti.potybus' 1; + 'com.soft373.taptaxi' 1; + 'com.kradac.ktxcore' 1; + '~^ru.crowdsystems.topcontrol.knd' 1; + 'pl.itaxi.driver' 1; + 'net.uztaxi.driver' 1; + 'OSMDroid/2.1 (its; rutaxi 3.28.0)' 1; + 'com.helleniccomms.mercedes.driver' 1; + 'ru.taximaster.www' 1; + 'com.arobs.trackgps' 1; + 'com.helleniccomms.asteras.driver' 1; + + # Malware + 'Agent Smith' 1; + # '~[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' 1; # Fake UA +} + +map $http_referer $denied_referer { + default 0; # Not denied + # Faked sites + 'http://www.openstreetmap.org/' 1; + 'http://www.openstreetmap.org' 1; + 'https://www.openstreetmap.org' 1; + 'http://openstreetmap.org/' 1; + 'http://openstreetmap.org' 1; + 'https://openstreetmap.org' 1; + 'http://www.osm.org/' 1; + 'http://www.osm.org' 1; + 'http://osm.org/' 1; + 'http://osm.org' 1; + 'http://google.com' 1; + 'http://www.google.com' 1; + 'http://google.com/' 1; + 'http://www.google.com/' 1; + 'https://google.com' 1; + 'https://www.google.com' 1; + 'https://google.com/' 1; + 'https://www.google.com/' 1; + 'http://www.microsoft.com/' 1; + + # Overusing websites + '~^https?://pmap\.kuku\.lu/' 1; + '~^https?://[^.]*\.pmap\.kuku\.lu/' 1; + '~^https?://fastpokemap\.com/' 1; + '~^https?://[^.]*\.fastpokemap\.com/' 1; + '~^https?://pkget\.com/' 1; + '~^https?://[^.]*\.pkget\.com/' 1; + '~^https?://twpkinfo\.com/' 1; + '~^https?://[^.]*\.twpkinfo\.com/' 1; + '~^https?://9db\.jp/' 1; + '~^https?://[^.]*\.9db\.jp/' 1; + '~^https?://clustrmaps\.com/' 1; + '~^https?://[^.]*\.clustrmaps\.com/' 1; + '~^https?://geoportal360\.pl/' 1; + '~^https?://skelbiu\.lt/' 1; + '~^https?://[^.]*\.skelbiu\.lt/' 1; + '~^https?://wialon\.[^.]*/' 1; # wialon has many domains, so block them all + '~^https?://[^.]*\.wialon\.[^.]*/' 1; + '~^https?://gps-trace\.com/' 1; + '~^https?://[^.]*\.gps-trace\.com/' 1; + '~^https?://cellmapper\.net/' 1; + '~^https?://[^.]*\.cellmapper\.net/' 1; +} + +map $http_referer $osm_referer { + default ''; # False + '~^https:\/\/www\.openstreetmap\.org\/' 'osm'; # True } # Limit Cache-Control header to only approved User-Agents -map $http_user_agent $limit_http_cache_control { - default ''; # Unset Header - '~^Mozilla\/5\.0\ QGIS\/' ''; # Unset Header - '~^Mozilla\/5\.0\ ' $http_cache_control; # Pass Header +map $tile_cache$osm_referer$http_user_agent $limit_http_cache_control { + default ''; # Unset Header + '~^clientosmMozilla\/5\.0\ \(X11' $http_cache_control; # Pass Header + '~^clientosmMozilla\/5\.0\ \(Windows' $http_cache_control; # Pass Header + '~^clientosmMozilla\/5\.0\ \(iPhone' $http_cache_control; # Pass Header + '~^clientosmMozilla\/5\.0\ \(Macintosh' $http_cache_control; # Pass Header + '~^clientosmMozilla\/5\.0\ \(Linux' $http_cache_control; # Pass Header } # Limit Pragma header to only approved User-Agents -map $http_user_agent $limit_http_pragma { - default ''; # Unset Header - '~^Mozilla\/5\.0\ QGIS\/' ''; # Unset Header - '~^Mozilla\/5\.0\ ' $http_pragma; # Pass Header +map $tile_cache$osm_referer$http_user_agent $limit_http_pragma { + default ''; # Unset Header + '~^clientosmMozilla\/5\.0\ \(X11' $http_pragma; # Pass Header + '~^clientosmMozilla\/5\.0\ \(Windows' $http_pragma; # Pass Header + '~^clientosmMozilla\/5\.0\ \(iPhone' $http_pragma; # Pass Header + '~^clientosmMozilla\/5\.0\ \(Macintosh' $http_pragma; # Pass Header + '~^clientosmMozilla\/5\.0\ \(Linux' $http_pragma; # Pass Header +} + +# Find Browser User-Agents which are not sending a referer. +# Browsers with no referer could be due to Browser extension or website Referrer-Policy +map $tile_cache$http_referer$scheme$http_user_agent $deny_missing_referer { + default 0; # Not denied + '~^clienthttpsMozilla\/5\.0\ \(X11' 1; + '~^clienthttpsMozilla\/5\.0\ \(Windows' 1; + '~^clienthttpsMozilla\/5\.0\ \(iPhone' 1; + '~^clienthttpsMozilla\/5\.0\ \(Macintosh' 1; + '~^clienthttpsMozilla\/5\.0\ \(Linux' 1; } server { - listen 443 ssl fastopen=2048 http2 default_server; + # IPv4 + listen 80 deferred backlog=16384 reuseport fastopen=2048 default_server; + listen 443 ssl deferred backlog=16384 reuseport fastopen=2048 http2 default_server; + # IPv6 + listen [::]:80 deferred backlog=16384 reuseport fastopen=2048 default_server; + listen [::]:443 ssl deferred backlog=16384 reuseport fastopen=2048 http2 default_server; server_name localhost; proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; ssl_certificate /etc/ssl/certs/tile.openstreetmap.org.pem; ssl_certificate_key /etc/ssl/private/tile.openstreetmap.org.key; + # Requests sent within early data are subject to replay attacks. + # See: http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_early_data + ssl_early_data on; + + # Immediately 404 layers we do not support +<% for i in 20..99 do %> + location /<%= i %>/ { + return 404; + } +<% end %> + + # Immediately 404 silly tile requests + location = /0/-1/-1.png { + return 404; + } + location = /0/-1/0.png { + return 404; + } + location = /0/-1/1.png { + return 404; + } + location = /0/0/-1.png { + return 404; + } + location = /0/0/1.png { + return 404; + } + location = /0/0/2.png { + return 404; + } + location = /0/1/-1.png { + return 404; + } + location = /0/1/0.png { + return 404; + } + location = /0/1/1.png { + return 404; + } + location = /0/1/2.png { + return 404; + } + location = /0/2/0.png { + return 404; + } + location = /0/2/1.png { + return 404; + } + location = /0/2/2.png { + return 404; + } + location = /1/-1/-1.png { + return 404; + } + location = /1/-1/0.png { + return 404; + } + location = /1/-1/1.png { + return 404; + } + location = /1/-1/2.png { + return 404; + } + location = /1/0/-1.png { + return 404; + } + location = /1/1/-1.png { + return 404; + } + location = /1/2/-1.png { + return 404; + } + location = /2/-1/0.png { + return 404; + } + location = /2/-1/1.png { + return 404; + } + location = /2/-1/2.png { + return 404; + } + location = /2/-1/3.png { + return 404; + } + location = /2/0/-1.png { + return 404; + } + location = /2/1/-1.png { + return 404; + } + location = /2/1/4.png { + return 404; + } + location = /2/2/4.png { + return 404; + } + location = /2/3/4.png { + return 404; + } + location = /2/4/0.png { + return 404; + } + location = /2/4/1.png { + return 404; + } + location = /2/4/2.png { + return 404; + } + location = /2/4/3.png { + return 404; + } + location = /2/4/4.png { + return 404; + } + +<% for i in 0..16 do %> +<% if i == 0 -%> + # Default Fallback Location Handler (lowest) location / { +<% elsif -%> + # Dedicated zoom handler for caching + location /<%= i %>/ { +<% end %> + # Only allow GET / HEAD / OPTIONS (CORS) requests + limit_except GET HEAD OPTIONS { + deny all; + } + proxy_pass http://tile_cache_backend; proxy_set_header X-Forwarded-For $remote_addr; proxy_http_version 1.1; proxy_set_header Connection ''; - proxy_connect_timeout 5s; + proxy_connect_timeout 20s; + + proxy_next_upstream_tries 2; + # Replace host header. + proxy_set_header Host 'tile.openstreetmap.org'; # Do not pass cookies to backends. proxy_set_header Cookie ''; # Do not pass Accept-Encoding to backends. proxy_set_header Accept-Encoding ''; + # Do not pass Accept to backends. + proxy_set_header Accept ''; + # Do not pass Accept-Language to backends as unused. + proxy_set_header Accept-Language ''; + proxy_set_header Accept-Charset ''; + # Do not send origin, we allow all. + proxy_set_header origin ''; + # Do not pass invalid headers to backend. + proxy_set_header X-Forwarded-Host ''; + proxy_set_header X-Host ''; + proxy_set_header Authorization ''; + proxy_set_header Proxy-Authorization ''; + + # Drop partial requests + proxy_set_header range ''; # Do not allow setting cookies from backends due to caching. proxy_ignore_headers Set-Cookie; proxy_hide_header Set-Cookie; +<% if i != 0 -%> + # Caching + proxy_cache "proxy_cache_zone"; + proxy_cache_lock on; + proxy_cache_valid 200 2d; + proxy_cache_valid 404 15m; + # Serve stale cache on errors or if updating + proxy_cache_use_stale error timeout updating http_404 http_500 http_503 http_504; + # If in cache as stale, serve stale and update in background + proxy_cache_background_update on; + # Workaround nginx async bug which causes stale cache replies to wait for the async backend cache update reply (seen in v1.16.0) + keepalive_requests 0; + # Enable revalidation using If-Modified-Since and If-None-Match for stale items + proxy_cache_revalidate on; + proxy_cache_min_uses 4; + + add_header x-cache-status "$upstream_cache_status - <%= node[:hostname] %>"; +<% else %> + # Severely rate limit Browser UAs which are not sending a referer. + # With no referer we do not know who is using tiles + if ($deny_missing_referer) { + set $limit_rate 1024; + add_header x-cache-ratelimit "missing-referer"; + } +<% end -%> + # Set a QoS cookie if none presented (uses nginx Map) - add_header Set-Cookie $cookie_qos_token_set; + # add_header Set-Cookie $cookie_qos_token_set; <% if node[:ssl][:strict_transport_security] -%> # Ensure Strict-Transport-Security header is removed from proxied server responses proxy_hide_header Strict-Transport-Security; @@ -81,11 +405,20 @@ server { <% end -%> # QoS Traffic Rate see $limit_rate on http://nginx.org/en/docs/http/ngx_http_core_module.html - set $limit_rate $limit_rate_qos; + # set $limit_rate $limit_rate_qos; # Allow Higher Traffic Rate from Approved User-Agents which do not support cookies (uses nginx Map) - if ($approved_scraper) { - set $limit_rate 32768; + # if ($approved_scraper) { + # set $limit_rate 65536; + # } + + if ($denied_scraper) { + set $limit_rate 512; + return 429; + } + if ($denied_referer) { + set $limit_rate 512; + return 418; } # Strip any ?query parameters from urls @@ -95,19 +428,5 @@ server { proxy_set_header Cache-Control $limit_http_cache_control; proxy_set_header Pragma $limit_http_pragma; } -} - -# Convert all http requests to https -server { - listen 80 default_server; - listen [::]:80 default_server; - server_name _; - return 301 https://$host$request_uri; -} - -server { - listen 80; - listen [::]:80; - server_name ~^(?(?:[a-d]\.)?tile)\.osm\.org$; - return 301 https://$subdomain.openstreetmap.org$request_uri; +<% end %> }