- sysfs
- taginfo
- tile
- - tilecache
- tilelog
- tools
- trac
- name: tile
run_list:
- recipe[tile::default]
- - name: tilecache
- run_list:
- - recipe[tilecache::default]
- name: tilelog
run_list:
- recipe[tilelog::default]
frontends = search(:node, "recipes:web\\:\\:frontend").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.sort_by(&:name).map do |n|
{ :name => n.name.split(".").first, :interface => n.interfaces(:role => :external).first[:interface].tr(".", "_") }
end
-tilecaches = search(:node, "roles:tilecache").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.sort_by(&:name).map do |n|
- { :name => n.name.split(".").first, :interface => n.interfaces(:role => :external).first[:interface].tr(".", "_") }
-end
renderers = search(:node, "roles:tile").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.sort_by(&:name).map do |n|
{ :name => n.name.split(".").first, :interface => n.interfaces(:role => :external).first[:interface].tr(".", "_") }
end
mode "644"
variables :expiry_time => expiry_time, :clients => clients,
:frontends => frontends, :geocoders => geocoders,
- :tilecaches => tilecaches, :renderers => renderers
+ :renderers => renderers
end
apache_module "fcgid"
memcached_multi_bytes.bytes_written.label Traffic in (-) / out (+)
memcached_multi_bytes.bytes_written.cdef bytes_written,8,*
<% end -%>
-<% unless @tilecaches.empty? -%>
-
-# Configure compound graphs for tile.openstreetmap.org
-[tile.openstreetmap.org]
- update no
- network_in.graph_title Inbound network traffic
- network_in.graph_vlabel bits in per ${graph_period}
- network_in.graph_category network
- network_in.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:if_%%interface%%.down", @tilecaches %>
- network_in.graph_total total
- network_in.graph_args --lower-limit 0
-<% @tilecaches.each do |tc| -%>
- network_in.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
- network_in.<%= tc[:name].tr("-", "_") %>.cdef <%= tc[:name].tr("-", "_") %>,8,*
- network_in.<%= tc[:name].tr("-", "_") %>.draw AREASTACK
- network_in.<%= tc[:name].tr("-", "_") %>.min 0
-<% end -%>
- network_out.graph_title Outbound network traffic
- network_out.graph_vlabel bits out per ${graph_period}
- network_out.graph_category network
- network_out.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:if_%%interface%%.up", @tilecaches %>
- network_out.graph_total total
- network_out.graph_args --lower-limit 0
-<% @tilecaches.each do |tc| -%>
- network_out.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
- network_out.<%= tc[:name].tr("-", "_") %>.cdef <%= tc[:name].tr("-", "_") %>,8,*
- network_out.<%= tc[:name].tr("-", "_") %>.draw AREASTACK
- network_out.<%= tc[:name].tr("-", "_") %>.min 0
-<% end -%>
- squid_delay_pools.graph_title IPs being delayed with referer
- squid_delay_pools.graph_args --base 1000 -l 0
- squid_delay_pools.graph_vlabel IPs
- squid_delay_pools.graph_order squid_delay1
- squid_delay_pools.graph_category squid
- squid_delay_pools.squid_delay1.sum <%= Chef::Munin.expand "%%name%%.openstreetmap.org:squid_delay_pools.squid_delay1", @tilecaches %>
- squid_delay_pools.squid_delay1.label IPs
- squid_delay_pools.squid_delay1.min 0
- squid_delay_pools.squid_delay1.draw AREA
- squid_delay_pools_noreferer.graph_title No-referer IPs being delayed
- squid_delay_pools_noreferer.graph_args --base 1000 -l 0
- squid_delay_pools_noreferer.graph_vlabel IPs
- squid_delay_pools_noreferer.graph_order squid_delay2
- squid_delay_pools_noreferer.graph_category squid
- squid_delay_pools_noreferer.squid_delay2.sum <%= Chef::Munin.expand "%%name%%.openstreetmap.org:squid_delay_pools_noreferer.squid_delay2", @tilecaches %>
- squid_delay_pools_noreferer.squid_delay2.label IPs
- squid_delay_pools_noreferer.squid_delay2.min 0
- squid_delay_pools_noreferer.squid_delay2.draw AREA
- squid_requests.graph_title Squid client requests
- squid_requests.graph_args --base 1000 -l 0
- squid_requests.graph_vlabel requests / ${graph_period}
- squid_requests.graph_order <%= Chef::Munin.expand "%%%name%%%_hits=%%name%%.openstreetmap.org:squid_requests.hits %%%name%%%_errors=%%name%%.openstreetmap.org:squid_requests.errors %%%name%%%_requests=%%name%%.openstreetmap.org:squid_requests.requests", @tilecaches %> hits=<%= @tilecaches.first[:name] %>.openstreetmap.org:squid_requests.hits errors=<%= @tilecaches.first[:name] %>.openstreetmap.org:squid_requests.errors requests=<%= @tilecaches.first[:name] %>.openstreetmap.org:squid_requests.requests
- squid_requests.graph_total total
- squid_requests.graph_category squid
-<% @tilecaches.each do |tc| -%>
- squid_requests.<%= tc[:name].tr("-", "_") %>_hits.graph no
- squid_requests.<%= tc[:name].tr("-", "_") %>_errors.graph no
- squid_requests.<%= tc[:name].tr("-", "_") %>_requests.graph no
-<% end -%>
- squid_requests.hits.cdef 0,<%= Chef::Munin.expand "%%%name%%%_hits", @tilecaches, ",+," %>,+
- squid_requests.hits.label hits
- squid_requests.hits.draw AREA
- squid_requests.errors.cdef 0,<%= Chef::Munin.expand "%%%name%%%_errors", @tilecaches, ",+," %>,+
- squid_requests.errors.label errors
- squid_requests.errors.draw STACK
- squid_requests.requests.cdef 0,<%= Chef::Munin.expand "%%%name%%%_requests", @tilecaches, ",+," %>,+,hits,-,errors,-
- squid_requests.requests.label misses
- squid_requests.requests.draw STACK
- squid_traffic.graph_title Squid traffic status
- squid_traffic.graph_args --base 1000
- squid_traffic.graph_vlabel bits per ${graph_period}
- squid_traffic.graph_order kbytes_in kbytes_out hit_kbytes_out
- squid_traffic.graph_category squid
- squid_traffic.kbytes_in.sum <%= Chef::Munin.expand "%%name%%.openstreetmap.org:squid_traffic.kbytes_in", @tilecaches %>
- squid_traffic.kbytes_in.label received
- squid_traffic.kbytes_in.cdef kbytes_in,8096,*
- squid_traffic.kbytes_out.sum <%= Chef::Munin.expand "%%name%%.openstreetmap.org:squid_traffic.kbytes_out", @tilecaches %>
- squid_traffic.kbytes_out.label sent
- squid_traffic.kbytes_out.cdef kbytes_out,8096,*
- squid_traffic.hit_kbytes_out.sum <%= Chef::Munin.expand "%%name%%.openstreetmap.org:squid_traffic.hit_kbytes_out", @tilecaches %>
- squid_traffic.hit_kbytes_out.label from cache
- squid_traffic.hit_kbytes_out.cdef hit_kbytes_out,8096,*
- squid_times_http.graph_title Squid Http Service Times
- squid_times_http.graph_category squid
- squid_times_http.graph_args --lower-limit 0
- squid_times_http.graph_vlabel median reponse times (s)
- squid_times_http.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:squid_times.mean_http", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_times_http.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-<% end -%>
- squid_times_cmis.graph_title Squid Cache Miss Service Times
- squid_times_cmis.graph_category squid
- squid_times_cmis.graph_args --lower-limit 0
- squid_times_cmis.graph_vlabel median reponse times (s)
- squid_times_cmis.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:squid_times.mean_cmis", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_times_cmis.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-<% end -%>
- squid_times_chits.graph_title Squid Cache Hit Service Times
- squid_times_chits.graph_category squid
- squid_times_chits.graph_args --lower-limit 0
- squid_times_chits.graph_vlabel median reponse times (s)
- squid_times_chits.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:squid_times.mean_chits", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_times_chits.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-<% end -%>
- squid_times_nhits.graph_title Squid Cache Near Hit Service Times
- squid_times_nhits.graph_category squid
- squid_times_nhits.graph_args --lower-limit 0
- squid_times_nhits.graph_vlabel median reponse times (s)
- squid_times_nhits.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:squid_times.mean_nhits", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_times_nhits.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-<% end -%>
- squid_times_nmr.graph_title Squid Cache Not Modified Service Times
- squid_times_nmr.graph_category squid
- squid_times_nmr.graph_args --lower-limit 0
- squid_times_nmr.graph_vlabel median reponse times (s)
- squid_times_nmr.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:squid_times.mean_nmr", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_times_nmr.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-<% end -%>
- squid_times_dnsl.graph_title Squid Cache DNS Lookup Service Times
- squid_times_dnsl.graph_category squid
- squid_times_dnsl.graph_args --lower-limit 0
- squid_times_dnsl.graph_vlabel median reponse times (s)
- squid_times_dnsl.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:squid_times.mean_dnsl", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_times_dnsl.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-<% end -%>
- squid_request_hitrates.graph_title Squid Cache Request Hit Rates
- squid_request_hitrates.graph_category squid
- squid_request_hitrates.graph_args --lower-limit 0 --upper-limit 100
- squid_request_hitrates.graph_vlabel %
- squid_request_hitrates.graph_order <%= Chef::Munin.expand "%%%name%%%_total=%%name%%.openstreetmap.org:squid_requests.requests %%%name%%%_hits=%%name%%.openstreetmap.org:squid_requests.hits", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_request_hitrates.<%= tc[:name].tr("-", "_") %>_total.graph no
- squid_request_hitrates.<%= tc[:name].tr("-", "_") %>_hits.cdef <%= tc[:name].tr("-", "_") %>_hits,<%= tc[:name].tr("-", "_") %>_total,/,100,*
- squid_request_hitrates.<%= tc[:name].tr("-", "_") %>_hits.label <%= tc[:name] %>
- squid_request_hitrates.<%= tc[:name].tr("-", "_") %>_hits.draw LINE1
-<% end -%>
- squid_byte_hitrates.graph_title Squid Cache Byte Hit Rates
- squid_byte_hitrates.graph_category squid
- squid_byte_hitrates.graph_args --lower-limit 0 --upper-limit 100
- squid_byte_hitrates.graph_vlabel %
- squid_byte_hitrates.graph_order <%= Chef::Munin.expand "%%%name%%%_total=%%name%%.openstreetmap.org:squid_traffic.kbytes_out %%%name%%%_hits=%%name%%.openstreetmap.org:squid_traffic.hit_kbytes_out", @tilecaches %>
-<% @tilecaches.each do |tc| -%>
- squid_byte_hitrates.<%= tc[:name].tr("-", "_") %>_total.graph no
- squid_byte_hitrates.<%= tc[:name].tr("-", "_") %>_hits.cdef <%= tc[:name].tr("-", "_") %>_hits,<%= tc[:name].tr("-", "_") %>_total,/,100,*
- squid_byte_hitrates.<%= tc[:name].tr("-", "_") %>_hits.label <%= tc[:name] %>
- squid_byte_hitrates.<%= tc[:name].tr("-", "_") %>_hits.draw LINE1
-<% end -%>
- nginx_requests.graph_title Nginx requests
- nginx_requests.graph_vlabel Requests per ${graph_period}
- nginx_requests.graph_category nginx
- nginx_requests.graph_order <%= Chef::Munin.expand "%%%name%%%=%%name%%.openstreetmap.org:nginx_request.request", @tilecaches %>
- nginx_requests.graph_total total
- nginx_requests.graph_args --lower-limit 0
-<% @tilecaches.each do |tc| -%>
- nginx_requests.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
- nginx_requests.<%= tc[:name].tr("-", "_") %>.draw AREASTACK
- nginx_requests.<%= tc[:name].tr("-", "_") %>.min 0
-<% end -%>
-<% end -%>
<% unless @renderers.empty? -%>
# Configure compound graphs for render.openstreetmap.org
<% node[:roles].sort.each do |role| -%>
chef_role{name="<%= role %>"} 1
<% end -%>
-<% if node[:roles].include?("tilecache") -%>
-# HELP chef_tile_parent Information about tile cache parents
-# TYPE chef_tile_parent gauge
-chef_tile_parent{name="<%= node[:tilecache][:tile_parent].split(".").first %>"} 1
-<% end -%>
<% node[:prometheus][:metrics].sort.each do |name, details| -%>
# HELP <%= name %> <%= details[:help] %>
# TYPE <%= name %> gauge
+++ /dev/null
-# tilecache cookbook
-
-This cookbook installs and configures the tile caches for the
-tile.openstreetmap.org tileservers.
+++ /dev/null
-default[:tilecache][:tile_parent] = "render.openstreetmap.org"
-
-# Per IP bucket refill rate
-default[:tilecache][:ip_bucket_refill] = 4096
-# Per IP bucket size
-default[:tilecache][:ip_bucket_size] = 67108864
-# Per Class C refill rate
-default[:tilecache][:net_bucket_refill] = 8192
-# Per Class C bucket size
-default[:tilecache][:net_bucket_size] = 134217728
-
-# Enable nginx cache
-default[:nginx][:cache][:proxy][:enable] = true
+++ /dev/null
-%{time_total},%{http_code},%{url_effective},%{time_namelookup},%{time_connect},%{time_appconnect},%{time_pretransfer},%{time_redirect},%{time_starttransfer}\n
+++ /dev/null
-name "tilecache"
-maintainer "OpenStreetMap Administrators"
-maintainer_email "admins@openstreetmap.org"
-license "Apache-2.0"
-description "Installs and configures a tile cache"
-
-version "1.0.0"
-supports "ubuntu"
-depends "fail2ban"
-depends "munin"
-depends "nginx"
-depends "ohai"
-depends "squid"
-depends "ssl"
+++ /dev/null
-#
-# Cookbook:: tilecache
-# Recipe:: default
-#
-# Copyright:: 2011, OpenStreetMap Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "ipaddr"
-
-include_recipe "fail2ban"
-include_recipe "munin"
-include_recipe "nginx"
-include_recipe "squid"
-include_recipe "ssl"
-
-package "apache2" do
- action :remove
-end
-
-package %w[
- curl
- xz-utils
- openssl
-]
-
-# oathtool for QoS token
-package "oathtool"
-
-tilecaches = search(:node, "roles:tilecache").sort_by { |n| n[:hostname] }
-tilerenders = search(:node, "roles:tile").sort_by { |n| n[:hostname] }
-
-web_passwords = data_bag_item("web", "passwords")
-
-tilecaches.each do |cache|
- cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address|
- firewall_rule "accept-squid" do
- action :accept
- family "inet"
- source "net:#{address}"
- dest "fw"
- proto "tcp:syn"
- dest_ports "3128"
- source_ports "1024:"
- end
-
- firewall_rule "accept-squid-icp" do
- action :accept
- family "inet"
- source "net:#{address}"
- dest "fw"
- proto "udp"
- dest_ports "3130"
- source_ports "3130"
- end
-
- firewall_rule "accept-squid-icp-reply" do
- action :accept
- family "inet"
- source "fw"
- dest "net:#{address}"
- proto "udp"
- dest_ports "3130"
- source_ports "3130"
- end
-
- firewall_rule "accept-squid-htcp" do
- action :accept
- family "inet"
- source "net:#{address}"
- dest "fw"
- proto "udp"
- dest_ports "4827"
- source_ports "4827"
- end
-
- firewall_rule "accept-squid-htcp-reply" do
- action :accept
- family "inet"
- source "fw"
- dest "net:#{address}"
- proto "udp"
- dest_ports "4827"
- source_ports "4827"
- end
- end
-end
-
-squid_fragment "tilecache" do
- template "squid.conf.erb"
- variables :caches => tilecaches, :renders => tilerenders
-end
-
-package "rsync"
-
-template "/etc/logrotate.d/squid" do
- source "logrotate.squid.erb"
- owner "root"
- group "root"
- mode "644"
-end
-
-nginx_site "default" do
- action [:delete]
-end
-
-template "/usr/local/bin/nginx_generate_tilecache_qos_map" do
- source "nginx_generate_tilecache_qos_map.erb"
- owner "root"
- group "root"
- mode "750"
- variables :totp_key => web_passwords["totp_key"]
-end
-
-cron_d "tilecache" do
- action :delete
-end
-
-cron_d "tilecache-generate-qos-map" do
- minute "0"
- user "root"
- command "/usr/local/bin/nginx_generate_tilecache_qos_map"
-end
-
-cron_d "tilecache-curl-time" do
- user "www-data"
- command "/srv/tilecache/tilecache-curl-time"
-end
-
-cron_d "tilecache-curl-time-cleanup" do
- minute "15"
- hour "0"
- user "www-data"
- command "/srv/tilecache/tilecache-curl-time-cleanup"
-end
-
-execute "execute_nginx_generate_tilecache_qos_map" do
- command "/usr/local/bin/nginx_generate_tilecache_qos_map"
- creates "/etc/nginx/conf.d/tile_qos_rates.map"
- action :run
-end
-
-ssl_certificate "tile.openstreetmap.org" do
- domains ["tile.openstreetmap.org",
- "a.tile.openstreetmap.org",
- "b.tile.openstreetmap.org",
- "c.tile.openstreetmap.org",
- "tile.osm.org",
- "a.tile.osm.org",
- "b.tile.osm.org",
- "c.tile.osm.org"]
- notifies :restart, "service[nginx]"
-end
-
-nginx_site "tile" do
- template "nginx_tile.conf.erb"
- variables :caches => tilecaches
-end
-
-template "/etc/logrotate.d/nginx" do
- source "logrotate.nginx.erb"
- owner "root"
- group "root"
- mode "644"
-end
-
-fail2ban_jail "squid" do
- maxretry 1000
-end
-
-tilerenders.each do |render|
- munin_plugin "ping_#{render[:fqdn]}" do
- target "ping_"
- conf "munin.ping.erb"
- conf_variables :host => render[:fqdn]
- end
-end
-
-directory "/srv/tilecache" do
- owner "root"
- group "root"
- mode "755"
-end
-
-directory "/srv/tilecache/data" do
- owner "www-data"
- group "www-data"
- mode "755"
-end
-
-cookbook_file "/srv/tilecache/tilecache-curl-time.txt" do
- source "tilecache-curl-time.txt"
- owner "root"
- group "root"
- mode "755"
-end
-
-template "/srv/tilecache/tilecache-curl-time" do
- source "tilecache-curl-time.erb"
- owner "root"
- group "root"
- mode "755"
- variables :caches => tilecaches, :renders => tilerenders
-end
-
-template "/srv/tilecache/tilecache-curl-time-cleanup" do
- source "tilecache-curl-time-cleanup.erb"
- owner "root"
- group "root"
- mode "755"
-end
-
-ohai_plugin "tilecache" do
- template "ohai.rb.erb"
-end
+++ /dev/null
-# DO NOT EDIT - This file is being maintained by Chef
-
-/var/log/nginx/*.log {
- daily
- missingok
- rotate 7
- compress
- delaycompress
- notifempty
- create 640 nginx adm
- sharedscripts
- postrotate
- [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
- endscript
-}
+++ /dev/null
-# DO NOT EDIT - This file is being maintained by Chef
-
-/var/log/squid/*.log {
- daily
- compress
- compresscmd /usr/bin/xz
- compressoptions --threads=<%= [ node[:cpu][:total] / 2, 1 ].max.ceil %>
- uncompresscmd /usr/bin/unxz
- compressext .xz
- rotate 2
- missingok
- nocreate
- sharedscripts
- postrotate
-<% if node[:lsb][:release].to_f < 20.04 -%>
- test ! -e /var/run/squid.pid || /usr/sbin/squid -k rotate
-<% else -%>
- test ! -e /run/squid/squid.pid || /usr/sbin/squid -k rotate
-<% end -%>
- endscript
- lastaction
- /usr/bin/rsync --preallocate /var/log/squid/access.log.1.xz ironbelly::logs/tile.openstreetmap.org/<%= node[:hostname] %>-`date -d "-1 days" +%Y-%m-%d`.xz || true
- endscript
-}
+++ /dev/null
-# DO NOT EDIT - This file is being maintained by Chef
-
-[ping_<%= @host %>]
-env.ping_args -c 1 -w 20
-env.ping_warn 0.5
-env.ping_crit 1.0
-env.packetloss_warn 10
-env.packetloss_crit 30
+++ /dev/null
-#!/bin/bash
-# DO NOT EDIT - This file is being maintained by Chef
-set -e
-
-NUM_TOKENS=4 # current + 4
-VALID_TOKEN=3600 # in seconds
-
-SECONDS_AGO=$((${NUM_TOKENS} * ${VALID_TOKEN}))
-OLD_TIME=$(/bin/date -u "+%Y-%m-%dT %H:%M:%S %z" -d "${SECONDS_AGO} seconds ago")
-QOS_TOKENS=($(/usr/bin/oathtool --totp --now="${OLD_TIME}" --window=${NUM_TOKENS} --time-step-size=${VALID_TOKEN}s -b "<%= @totp_key %>"))
-
-# ${qos_tokens[4]/[-1] } = OSM.org exclusive / current
-# ${qos_tokens[3]/[-2] } = OSM.org exclusive / stale
-# ${qos_tokens[2]/[-3] } = tile.openstreetmap.org default
-# ${qos_tokens[1]/[-4] } = stale ~ 1 hour
-# ${qos_tokens[0]} = expired
-
-# Test if number of tokens returned by oathtool is expected number
-if [ "${#QOS_TOKENS[@]}" -ne "$((${NUM_TOKENS}+1))" ]; then
- >&2 echo "ERROR: Unexpected number of tokens"
- exit 1
-fi
-
-QOS_TOKEN_OSM=${QOS_TOKENS[-1]} # Cookie set by openstreetmap.org
-QOS_TOKEN_OSM_STALE=${QOS_TOKENS[-2]} # Cookie set by openstreetmap.org stale
-QOS_TOKEN_DEFAULT=${QOS_TOKENS[-3]} # Cookie presented by tile.openstreetmap.org to browsers
-QOS_TOKEN_STALE=${QOS_TOKENS[-4]} # Cookie which has become stale and will be replaced
-
-if [ -z "$QOS_TOKEN_OSM" -o -z "$QOS_TOKEN_DEFAULT" -o -z "$QOS_TOKEN_STALE" ]; then
- >&2 echo "ERROR: Unexpected blank token"
- exit 2
-fi
-
-cat <<EOF >/etc/nginx/conf.d/tile_qos_rates.map
-default 8192; # Default Rate (No QoS cookie)
-"${QOS_TOKEN_STALE}" 24576; # Stale
-"${QOS_TOKEN_DEFAULT}" 24576; # Default
-"${QOS_TOKEN_OSM_STALE}" 32768; # Exclusive Stale
-"${QOS_TOKEN_OSM}" 32768; # Exclusive
-EOF
-
-cat <<EOF >/etc/nginx/conf.d/tile_qos_cookies.map
-default '_osm_totp_token=${QOS_TOKEN_DEFAULT}; Max-Age=${VALID_TOKEN}; Domain=openstreetmap.org; Path=/'; # Cookie Domain per RFC6265
-"${QOS_TOKEN_DEFAULT}" ''; # Do not Set-Cookie. # Default
-"${QOS_TOKEN_OSM_STALE}" ''; # Do not Set-Cookie. # Exclusive Stale
-"${QOS_TOKEN_OSM}" ''; # Do not Set-Cookie. # Exclusive
-EOF
-
-# Check config, reload config and fail safe
-# /etc/init.d/nginx configtest 2>/dev/null && /bin/systemctl try-reload-or-restart nginx
+++ /dev/null
-# DO NOT EDIT - This file is being maintained by Chef
-
-upstream tile_cache_backend {
- server 127.0.0.1:8080 weight=1000 max_fails=32;
- server 127.0.0.2:8080 weight=1000 max_fails=32;
-
- # Add the tile_siblings caches to relieve pressure if local squid failing
- # Balancer: round-robin
-<% server_weight = 1000 -%>
-<% Array(@node[:tilecache][:tile_siblings]).each do |cache_peer| -%>
-<% @caches.each do |cache| -%>
-<% if cache_peer == cache[:fqdn] -%>
-<% if cache[:hostname] != node[:hostname] -%>
-<% cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%>
- server <%= address %>:80 weight=<%= server_weight %> max_fails=32 backup; # Server <%= cache[:hostname] %>
-<% server_weight -= server_weight.div(2) -%>
-<% end -%>
-<% end -%>
-<% end -%>
-<% end -%>
-<% end -%>
-
- keepalive 128;
-}
-
-# Geo Map of tile caches
-geo $tile_cache {
- default "client";
-<% @caches.each do |cache| -%>
-<% cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%>
- <%= address %> "cache"; # <%= cache[:hostname] %>
-<% end -%>
-<% end -%>
-}
-
-# Rates table based on current cookie value
-# map $cookie__osm_totp_token $limit_rate_qos {
-# include /etc/nginx/conf.d/tile_qos_rates.map;
-# }
-
-# Set-Cookie table based on current cookie value
-# map $cookie__osm_totp_token $cookie_qos_token_set {
-# include /etc/nginx/conf.d/tile_qos_cookies.map;
-# }
-
-map $http_user_agent $approved_scraper {
- default 0; # Not approved
- '~^JOSM\/' 1; # JOSM
- '~^Mozilla\/5\.0\ QGIS\/' 1; # QGIS
-}
-
-map $http_user_agent $denied_scraper {
- default 0; # Not denied
- '' 1; # No User-Agent Set
- '-' 1;
-
- # Library defaults
- '~^Python\-urllib\/' 1;
- '~^python\-requests\/' 1;
- '~^node\-fetch\/' 1;
- '~^R$' 1;
- '~^Java\/' 1;
- '~^tiles$' 1;
- '~^okhttp\/' 1;
- '~^Microsoft-ATL-Native\/' 1;
- '/n software IPWorks HTTP/S Component - www.nsoftware.com' 1;
- '~^Wget\/' 1;
- 'java' 1;
-
- # Library defaults or fakes
- 'Android' 1;
- 'kc_android' 1;
- 'host' 1;
- '~^maptestapp' 1;
- 'Other' 1;
- 'osmdroid' 1;
- '~^tilelive-http' 1;
- '~^Java-http-client' 1;
-
- # Fakes
- 'Mozilla/4.0' 1;
- 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)' 1;
-
- # Bulk downloaders
- 'C# TilesDownloader' 1;
- 'MapDownloader' 1;
- '~^staticmaps' 1;
-
- # Overusage apps
- '~^runtastic' 1;
- '~^Where\ my\ children' 1;
- 'nossoonibusjp.android.crosswalk' 1;
- 'br.com.concisoti.potybus' 1;
- 'com.soft373.taptaxi' 1;
- 'com.kradac.ktxcore' 1;
- '~^ru.crowdsystems.topcontrol.knd' 1;
- 'pl.itaxi.driver' 1;
- 'net.uztaxi.driver' 1;
- 'OSMDroid/2.1 (its; rutaxi 3.28.0)' 1;
- 'com.helleniccomms.mercedes.driver' 1;
- 'ru.taximaster.www' 1;
- 'com.arobs.trackgps' 1;
- 'com.helleniccomms.asteras.driver' 1;
-
- # Malware
- 'Agent Smith' 1;
- # '~[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' 1; # Fake UA
-}
-
-map $http_referer $denied_referer {
- default 0; # Not denied
- # Faked sites
- 'http://www.openstreetmap.org/' 1;
- 'http://www.openstreetmap.org' 1;
- 'https://www.openstreetmap.org' 1;
- 'http://openstreetmap.org/' 1;
- 'http://openstreetmap.org' 1;
- 'https://openstreetmap.org' 1;
- 'http://www.osm.org/' 1;
- 'http://www.osm.org' 1;
- 'http://osm.org/' 1;
- 'http://osm.org' 1;
- 'http://google.com' 1;
- 'http://www.google.com' 1;
- 'http://google.com/' 1;
- 'http://www.google.com/' 1;
- 'https://google.com' 1;
- 'https://www.google.com' 1;
- 'https://google.com/' 1;
- 'https://www.google.com/' 1;
- 'http://www.microsoft.com/' 1;
-
- # Overusing websites
- '~^https?://pmap\.kuku\.lu/' 1;
- '~^https?://[^.]*\.pmap\.kuku\.lu/' 1;
- '~^https?://fastpokemap\.com/' 1;
- '~^https?://[^.]*\.fastpokemap\.com/' 1;
- '~^https?://pkget\.com/' 1;
- '~^https?://[^.]*\.pkget\.com/' 1;
- '~^https?://twpkinfo\.com/' 1;
- '~^https?://[^.]*\.twpkinfo\.com/' 1;
- '~^https?://9db\.jp/' 1;
- '~^https?://[^.]*\.9db\.jp/' 1;
- '~^https?://clustrmaps\.com/' 1;
- '~^https?://[^.]*\.clustrmaps\.com/' 1;
- '~^https?://geoportal360\.pl/' 1;
- '~^https?://skelbiu\.lt/' 1;
- '~^https?://[^.]*\.skelbiu\.lt/' 1;
- '~^https?://wialon\.[^.]*/' 1; # wialon has many domains, so block them all
- '~^https?://[^.]*\.wialon\.[^.]*/' 1;
- '~^https?://gps-trace\.com/' 1;
- '~^https?://[^.]*\.gps-trace\.com/' 1;
- '~^https?://cellmapper\.net/' 1;
- '~^https?://[^.]*\.cellmapper\.net/' 1;
-}
-
-map $http_referer $censored_referer {
- default 0; # Not denied
- # Blocked on board instructions
- '~^https?://schiebt-sie-ab\.de/' 1;
- '~^https?://[^.]*\.schiebt-sie-ab\.de/' 1;
-}
-
-
-map $http_referer $osm_referer {
- default ''; # False
- '~^https:\/\/www\.openstreetmap\.org\/' 'osm'; # True
-}
-
-# Limit Cache-Control header to only approved User-Agents
-map $tile_cache$osm_referer$http_user_agent $limit_http_cache_control {
- default ''; # Unset Header
- '~^clientosmMozilla\/5\.0\ \(X11' $http_cache_control; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(Windows' $http_cache_control; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(iPhone' $http_cache_control; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(Macintosh' $http_cache_control; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(Linux' $http_cache_control; # Pass Header
-}
-
-# Limit Pragma header to only approved User-Agents
-map $tile_cache$osm_referer$http_user_agent $limit_http_pragma {
- default ''; # Unset Header
- '~^clientosmMozilla\/5\.0\ \(X11' $http_pragma; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(Windows' $http_pragma; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(iPhone' $http_pragma; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(Macintosh' $http_pragma; # Pass Header
- '~^clientosmMozilla\/5\.0\ \(Linux' $http_pragma; # Pass Header
-}
-
-# Find Browser User-Agents which are not sending a referer.
-# Browsers with no referer could be due to Browser extension or website Referrer-Policy
-map $tile_cache$http_referer$scheme$http_user_agent $deny_missing_referer {
- default 0; # Not denied
- '~^clienthttpsMozilla\/5\.0\ \(X11' 1;
- '~^clienthttpsMozilla\/5\.0\ \(Windows' 1;
- '~^clienthttpsMozilla\/5\.0\ \(iPhone' 1;
- '~^clienthttpsMozilla\/5\.0\ \(Macintosh' 1;
- '~^clienthttpsMozilla\/5\.0\ \(Linux' 1;
-}
-
-server {
- # IPv4
- listen 80 deferred backlog=16384 reuseport default_server;
- listen 443 ssl deferred backlog=16384 reuseport http2 default_server;
- # IPv6
- listen [::]:80 deferred backlog=16384 reuseport default_server;
- listen [::]:443 ssl deferred backlog=16384 reuseport http2 default_server;
- server_name localhost;
-
- proxy_buffers 8 64k;
- proxy_busy_buffers_size 64k;
-
- ssl_certificate /etc/ssl/certs/tile.openstreetmap.org.pem;
- ssl_certificate_key /etc/ssl/private/tile.openstreetmap.org.key;
-
- # Requests sent within early data are subject to replay attacks.
- # See: http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_early_data
- ssl_early_data on;
-
- # Immediately 404 layers we do not support
-<% for i in 20..99 do %>
- location /<%= i %>/ {
- return 404;
- }
-<% end %>
-
- # Immediately 404 silly tile requests
- location = /0/-1/-1.png {
- return 404;
- }
- location = /0/-1/0.png {
- return 404;
- }
- location = /0/-1/1.png {
- return 404;
- }
- location = /0/0/-1.png {
- return 404;
- }
- location = /0/0/1.png {
- return 404;
- }
- location = /0/0/2.png {
- return 404;
- }
- location = /0/1/-1.png {
- return 404;
- }
- location = /0/1/0.png {
- return 404;
- }
- location = /0/1/1.png {
- return 404;
- }
- location = /0/1/2.png {
- return 404;
- }
- location = /0/2/0.png {
- return 404;
- }
- location = /0/2/1.png {
- return 404;
- }
- location = /0/2/2.png {
- return 404;
- }
- location = /1/-1/-1.png {
- return 404;
- }
- location = /1/-1/0.png {
- return 404;
- }
- location = /1/-1/1.png {
- return 404;
- }
- location = /1/-1/2.png {
- return 404;
- }
- location = /1/0/-1.png {
- return 404;
- }
- location = /1/1/-1.png {
- return 404;
- }
- location = /1/2/-1.png {
- return 404;
- }
- location = /2/-1/0.png {
- return 404;
- }
- location = /2/-1/1.png {
- return 404;
- }
- location = /2/-1/2.png {
- return 404;
- }
- location = /2/-1/3.png {
- return 404;
- }
- location = /2/0/-1.png {
- return 404;
- }
- location = /2/1/-1.png {
- return 404;
- }
- location = /2/1/4.png {
- return 404;
- }
- location = /2/2/4.png {
- return 404;
- }
- location = /2/3/4.png {
- return 404;
- }
- location = /2/4/0.png {
- return 404;
- }
- location = /2/4/1.png {
- return 404;
- }
- location = /2/4/2.png {
- return 404;
- }
- location = /2/4/3.png {
- return 404;
- }
- location = /2/4/4.png {
- return 404;
- }
-
-<% for i in 0..16 do %>
-<% if i == 0 -%>
- # Default Fallback Location Handler (lowest)
- location / {
-<% elsif -%>
- # Dedicated zoom handler for caching
- location /<%= i %>/ {
-<% end %>
- # Only allow GET / HEAD / OPTIONS (CORS) requests
- limit_except GET HEAD OPTIONS {
- deny all;
- }
-
- proxy_pass http://tile_cache_backend;
- proxy_set_header X-Forwarded-For $remote_addr;
- proxy_http_version 1.1;
- proxy_set_header Connection '';
-
- proxy_connect_timeout 20s;
-
- proxy_next_upstream_tries 2;
-
- # Replace host header.
- proxy_set_header Host 'tile.openstreetmap.org';
- # Drop all request headers and request body
- proxy_pass_request_headers off;
- proxy_pass_request_body off;
-
- # Do not allow setting cookies from backends due to caching.
- proxy_ignore_headers Set-Cookie;
- proxy_hide_header Set-Cookie;
-
-<% if i != 0 -%>
- # Caching
- proxy_cache "proxy_cache_zone";
- proxy_cache_lock on;
- proxy_cache_valid 200 2d;
- proxy_cache_valid 404 15m;
- # Serve stale cache on errors or if updating
- proxy_cache_use_stale error timeout updating http_404 http_500 http_503 http_504;
- # If in cache as stale, serve stale and update in background
- proxy_cache_background_update on;
- # Workaround nginx async bug which causes stale cache replies to wait for the async backend cache update reply (seen in v1.16.0)
- keepalive_requests 0;
- # Enable revalidation using If-Modified-Since and If-None-Match for stale items
- proxy_cache_revalidate on;
- proxy_cache_min_uses 4;
-
- add_header x-cache-status "$upstream_cache_status - <%= node[:hostname] %>";
-<% else %>
- # Severely rate limit Browser UAs which are not sending a referer.
- # With no referer we do not know who is using tiles
- if ($deny_missing_referer) {
- set $limit_rate 1024;
- add_header x-cache-ratelimit "missing-referer";
- }
-<% end -%>
-
- # Set a QoS cookie if none presented (uses nginx Map)
- # add_header Set-Cookie $cookie_qos_token_set;
-<% if node[:ssl][:strict_transport_security] -%>
- # Ensure Strict-Transport-Security header is removed from proxied server responses
- proxy_hide_header Strict-Transport-Security;
-
- # Enable HSTS
- add_header Strict-Transport-Security "<%= node[:ssl][:strict_transport_security] %>" always;
-<% end -%>
-
- # QoS Traffic Rate see $limit_rate on http://nginx.org/en/docs/http/ngx_http_core_module.html
- # set $limit_rate $limit_rate_qos;
-
- # Allow Higher Traffic Rate from Approved User-Agents which do not support cookies (uses nginx Map)
- # if ($approved_scraper) {
- # set $limit_rate 65536;
- # }
-
- if ($denied_scraper) {
- set $limit_rate 512;
- return 429;
- }
- if ($denied_referer) {
- set $limit_rate 512;
- return 418;
- }
-
- if ($censored_referer) {
- set $limit_rate 512;
- return 451 "Unavailable at OSMF Board request";
- }
-
- # Strip any ?query parameters from urls
- set $args '';
-
- # Allow cache purging headers only from select User-Agents (uses nginx Map)
- proxy_set_header Cache-Control $limit_http_cache_control;
- proxy_set_header Pragma $limit_http_pragma;
- }
-<% end %>
-}
+++ /dev/null
-Ohai.plugin(:TileCache) do
- provides "tilecache"
-
- def tile_siblings
- recent = Time.now - 600
- times = Hash.new
-
- # Find performance reports for last few minutes
- # Add up total time taken to download tile grouped by remote server
- # Remove 1 second per successful time report (de-prioritise new servers)
- # Add 10 seconds per failed time report request
- Dir.glob("/srv/tilecache/data/**/tilecache-*.txt").each do |path|
- if File.mtime(path) > recent
- IO.readlines(path).reverse.take(20).each do |sample|
- if sample =~ %r{^(\d+\.\d+),(\d+),https://([^/]+)/} then
- time = Regexp.last_match(1).to_f
- status = Regexp.last_match(2).to_i
- host = Regexp.last_match(3)
-
- if status == 200 then
- times[host] = times.fetch(host, 0) + (time + 1) * (time + 1) - 1
- else
- times[host] = times.fetch(host, 0) + 10
- end
- end
- end
- end
- end
-
- # Sort time reports
- # Strip to best 4 server names
- times.to_a.sort_by(&:last).take(4).map(&:first)
- end
-
- collect_data(:default) do
- tilecache Mash.new
-
- tilecache[:tile_siblings] = tile_siblings
- end
-end
+++ /dev/null
-acl osmtile_thishost dstdomain <%= node.name %>
-acl osmtile_sites dstdomain <%= node.name %> a.tile.openstreetmap.org b.tile.openstreetmap.org c.tile.openstreetmap.org tile.openstreetmap.org a.tile.osm.org b.tile.osm.org c.tile.osm.org tile.osm.org
-acl osmtiles_png urlpath_regex .png$
-
-acl whitelist_path urlpath_regex ^/cgi-bin/(export|debug)
-acl blacklist_path urlpath_regex ^/cgi-bin/
-acl blacklist_path urlpath_regex ^/MyAdmin/
-acl blacklist_path urlpath_regex ^/myadmin/
-acl blacklist_path urlpath_regex ^/pma/
-acl blacklist_path urlpath_regex ^/phpmyadmin/
-acl blacklist_path urlpath_regex ^/phpMyAdmin/
-acl blacklist_path urlpath_regex ^/idssvc/
-acl blacklist_path urlpath_regex ^/iesvc/
-acl blacklist_path urlpath_regex ^/invoker/
-acl blacklist_path urlpath_regex ^/jmx-console/
-acl blacklist_path urlpath_regex ^/manager/
-acl blacklist_path urlpath_regex ^/service/
-acl blacklist_path urlpath_regex ^/web-console/
-acl blacklist_path urlpath_regex ^/wstats/
-acl blacklist_path urlpath_regex ^/zecmd/
-
-http_access allow osmtile_sites whitelist_path
-http_access deny blacklist_path
-
-acl requestMethodGet method GET
-
-http_access allow osmtile_sites requestMethodGet
-
-acl osmtile_nocache_url urlpath_regex \.png/(status|dirty)$
-cache deny osmtile_sites osmtile_nocache_url
-
-<% @caches.each do |cache| -%>
-<% cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%>
-acl tile_caches src <%= address %>
-<% end -%>
-<% end -%>
-
-# Primary Parent
-<% if node[:squid][:version] < 4 -%>
-cache_peer <%= node[:tilecache][:tile_parent] %> parent 443 0 no-query originserver name=osmtileAccel login=PASS connect-timeout=120 no-digest weight=1000 ssl ssldomain=render.openstreetmap.org
-<% elsif node[:lsb][:release].to_f < 20.04 -%>
-cache_peer <%= node[:tilecache][:tile_parent] %> parent 443 0 no-query originserver name=osmtileAccel login=PASS connect-timeout=120 no-digest weight=1000 tls tlsdomain=render.openstreetmap.org
-<% else -%>
-cache_peer <%= node[:tilecache][:tile_parent] %> parent 443 0 no-query originserver name=osmtileAccel login=PASS connect-timeout=120 no-digest weight=1000 tls tlsdomain=render.openstreetmap.org tls-options=NORMAL:-VERS-TLS1.3
-<% end -%>
-cache_peer_access osmtileAccel allow osmtile_sites
-
-# Backup Parents
-<% @renders.each do |renders| -%>
-<% if node[:squid][:version] < 4 -%>
-cache_peer <%= renders[:hostname] %>.render.openstreetmap.org parent 443 0 no-query originserver name=osmtileAccelBackup<%= renders[:hostname] %> login=PASS connect-timeout=60 no-digest weight=10 ssl ssldomain=render.openstreetmap.org
-<% elsif node[:lsb][:release].to_f < 20.04 -%>
-cache_peer <%= renders[:hostname] %>.render.openstreetmap.org parent 443 0 no-query originserver name=osmtileAccelBackup<%= renders[:hostname] %> login=PASS connect-timeout=60 no-digest weight=10 tls tlsdomain=render.openstreetmap.org
-<% else -%>
-cache_peer <%= renders[:hostname] %>.render.openstreetmap.org parent 443 0 no-query originserver name=osmtileAccelBackup<%= renders[:hostname] %> login=PASS connect-timeout=60 no-digest weight=10 tls tlsdomain=render.openstreetmap.org tls-options=NORMAL:-VERS-TLS1.3
-<% end -%>
-cache_peer_access osmtileAccelBackup<%= renders[:hostname] %> allow osmtile_sites
-<% end -%>
-
-# ----------------------------------
-
-<% if node[:squid][:version] < 4 -%>
-
-#Allow tile_caches HTCP access
-htcp_access allow tile_caches
-
-#Allow tile_caches ICP access
-icp_access allow tile_caches
-<% end %>
-
-#----------------------------------
+++ /dev/null
-#/bin/bash
-set -eu
-/usr/bin/find /srv/tilecache/data/ -type f -iname '*.txt' -mtime +7 -print0 | /usr/bin/xargs -0 -n 12 --no-run-if-empty -P 2 /usr/bin/xz -9e
+++ /dev/null
-#!/bin/bash
-sleep $[ ( $RANDOM % 20 ) + 1 ]
-mkdir -p /srv/tilecache/data/$(date --utc "+%Y/%m")
-# localhost
-curl -w "@/srv/tilecache/tilecache-curl-time.txt" -o /dev/null -s -k -4 \
---max-time 60 \
-'http://localhost:8080/19/262106/174485.png' \
--H 'authority: c.tile.openstreetmap.org' \
--H 'sec-fetch-dest: image' \
--H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36' \
--H 'dnt: 1' \
--H 'accept: image/webp,image/apng,image/*,*/*;q=0.8' \
--H 'sec-fetch-site: same-site' \
--H 'sec-fetch-mode: no-cors' \
--H 'referer: https://www.openstreetmap.org/' \
--H 'accept-language: en-GB,en-US;q=0.9,en;q=0.8' \
---compressed >> /srv/tilecache/data/$(date --utc "+%Y/%m")/localhost-<%= node.name.split(".").first %>-$(date --utc "+%Y-%m-%dT%H").txt
-# render
-<% @renders.each do |render| -%>
-<% if render.name != node.name -%>
-<% render.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%>
- curl -w "@/srv/tilecache/tilecache-curl-time.txt" -o /dev/null -s -k -4 \
- --max-time 60 \
- 'https://<%= render.name %>/19/262106/174485.png' \
- -H 'authority: c.tile.openstreetmap.org' \
- -H 'sec-fetch-dest: image' \
- -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36' \
- -H 'dnt: 1' \
- -H 'accept: image/webp,image/apng,image/*,*/*;q=0.8' \
- -H 'sec-fetch-site: same-site' \
- -H 'sec-fetch-mode: no-cors' \
- -H 'referer: https://www.openstreetmap.org/' \
- -H 'accept-language: en-GB,en-US;q=0.9,en;q=0.8' \
- --compressed >> /srv/tilecache/data/$(date --utc "+%Y/%m")/render-<%= render.name.split(".").first %>-$(date --utc "+%Y-%m-%dT%H").txt
-<% end -%>
-<% end -%>
-<% end -%>
-# caches
-<% @caches.each do |cache| -%>
-<% if cache.name != node.name -%>
-<% cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| -%>
- curl -w "@/srv/tilecache/tilecache-curl-time.txt" -o /dev/null -s -k -4 \
- --max-time 60 \
- 'https://<%= cache.name %>/19/262106/174485.png' \
- -H 'authority: c.tile.openstreetmap.org' \
- -H 'sec-fetch-dest: image' \
- -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36' \
- -H 'dnt: 1' \
- -H 'accept: image/webp,image/apng,image/*,*/*;q=0.8' \
- -H 'sec-fetch-site: same-site' \
- -H 'sec-fetch-mode: no-cors' \
- -H 'referer: https://www.openstreetmap.org/' \
- -H 'accept-language: en-GB,en-US;q=0.9,en;q=0.8' \
- --compressed >> /srv/tilecache/data/$(date --utc "+%Y/%m")/tilecache-<%= cache.name.split(".").first %>-$(date --utc "+%Y-%m-%dT%H").txt
-<% end -%>
-<% end -%>
-<% end -%>
description "Master role applied to angor"
default_attributes(
- :hardware => {
- :shm_size => "18g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
:gateway => "2001:43f8:1f4:b00::1"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "16384 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 12800 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 16000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 22400 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 22800 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "capetown.render.openstreetmap.org"
}
)
run_list(
"role[inxza]",
- "role[tilecache]",
"role[ftp]"
)
description "Master role applied to ascalon"
default_attributes(
- :hardware => {
- :shm_size => "20g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
:gateway => "184.107.48.225"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "16384 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "montreal.render.openstreetmap.org"
}
)
run_list(
- "role[netalerts]",
- "role[tilecache]"
+ "role[netalerts]"
)
:zelja => { :status => :administrator }
}
},
- :hardware => {
- :shm_size => "18g"
- },
:location => "Osijek, Croatia",
:networking => {
:interfaces => {
:gateway => "fe80::161:53:30:97"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "16384 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "osijek.render.openstreetmap.org"
}
)
run_list(
- "role[carnet]",
- "role[tilecache]"
+ "role[carnet]"
)
description "Master role applied to firnen"
default_attributes(
- :hardware => {
- :shm_size => "36g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
:gateway => "188.241.28.81"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "32768 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "germany.render.openstreetmap.org"
}
)
run_list(
- "role[epix]",
- "role[tilecache]"
+ "role[epix]"
)
:hardware => {
:ipmi => {
:excluded_sensors => [3, 4]
- },
- :shm_size => "20g"
+ }
},
:munin => {
:plugins => {
"block/md0/md/sync_speed_max" => "100000"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "16384 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :nginx => {
- :cache => {
- :proxy => {
- :directory => "/store/nginx-cache/proxy-cache",
- :max_size => "32768M"
- }
- }
- },
- :tilecache => {
- :tile_parent => "moscow.render.openstreetmap.org"
}
)
run_list(
- "role[yandex]",
- "role[tilecache]"
+ "role[yandex]"
)
"2001:41c9:2:d6::/64", # bytemark external
"127.0.0.0/8", # localhost
"::1" # localhost
- ],
- :nodes_allow => "roles:tilecache"
+ ]
}
}
}
description "Master role applied to ladon"
default_attributes(
- :hardware => {
- :shm_size => "36g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
:gateway => "2001:648:2ffe:4::1"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "32768 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 12800 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 16000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 22400 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 22800 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "athens.render.openstreetmap.org"
}
)
run_list(
- "role[grnet]",
- "role[tilecache]"
+ "role[grnet]"
)
description "Master role applied to meraxes"
default_attributes(
- :hardware => {
- :shm_size => "36g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
:gateway => "2001:bc8:2::2:258:1"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "32768 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 12800 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 16000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 22400 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 22800 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "france.render.openstreetmap.org"
}
)
run_list(
- "role[scaleway]",
- "role[tilecache]"
+ "role[scaleway]"
)
description "Master role applied to neak"
default_attributes(
- :hardware => {
- :shm_size => "14g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
:gateway => "89.234.177.129"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "10240 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 12800 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 16000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 22400 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 22800 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "france.render.openstreetmap.org"
}
)
run_list(
- "role[faimaison]",
- "role[tilecache]"
+ "role[faimaison]"
)
description "Master role applied to nepomuk"
default_attributes(
- :hardware => {
- :shm_size => "14g"
- },
:networking => {
:firewall => {
:inet => [
}
}
},
- :squid => {
- :version => 4,
- :cache_mem => "10240 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
:sysfs => {
:hdd_tune => {
:comment => "Tune the queue for improved performance",
"block/vda/queue/nr_requests" => "128"
}
}
- },
- :tilecache => {
- :tile_parent => "france.render.openstreetmap.org"
}
)
run_list(
- "role[lyonix]",
- "role[tilecache]"
+ "role[lyonix]"
)
:hardware => {
:ipmi => {
:excluded_sensors => [19, 20, 21, 22]
- },
- :shm_size => "10g"
+ }
},
:munin => {
:plugins => {
"block/md1/md/sync_speed_max" => "100000"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "8192 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "oslo.render.openstreetmap.org"
}
)
run_list(
"role[blix-no]",
- "role[geodns]",
- "role[tilecache]"
+ "role[geodns]"
)
}
},
:private_address => "10.0.16.200"
- },
- :tilecache => {
- :tile_parent => "corvallis.render.openstreetmap.org"
}
)
description "Master role applied to takhisis"
default_attributes(
- :hardware => {
- :shm_size => "14g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
:gateway => "fe80::225:90ff:fe5d:c1e1"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "10240 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :nginx => {
- :cache => {
- :proxy => {
- :directory => "/store/nginx-cache/proxy-cache",
- :max_size => "65536M"
- }
- }
- },
- :tilecache => {
- :tile_parent => "netherlands.render.openstreetmap.org"
}
)
run_list(
- "role[tuxis]",
- "role[tilecache]"
+ "role[tuxis]"
)
+++ /dev/null
-name "tilecache"
-description "Role applied to all tile cache servers"
-
-default_attributes(
- :accounts => {
- :groups => {
- :proxy => {
- :members => [:tomh, :grant, :matt, :jburgess]
- }
- }
- },
- :nginx => {
- :access_log => false
- },
- :sysctl => {
- :sockets => {
- :comment => "Increase size of connection queue",
- :parameters => {
- "net.core.somaxconn" => 10000
- }
- },
- :network_conntrack_time_wait => {
- :comment => "Only track completed connections for 30 seconds",
- :parameters => {
- "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30"
- }
- },
- :network_conntrack_max => {
- :comment => "Increase max number of connections tracked",
- :parameters => {
- "net.netfilter.nf_conntrack_max" => "524288"
- }
- },
- :network_local_port_range => {
- :comment => "Increase available local port range",
- :parameters => {
- "net.ipv4.ip_local_port_range" => "1024\t65535"
- }
- },
- :network_tcp_timewait_reuse => {
- :comment => "Allow tcp timewait reuse",
- :parameters => {
- "net.ipv4.tcp_tw_reuse" => 1
- }
- },
- :squid_swappiness => {
- :comment => "Prefer not to swapout to free memory",
- :parameters => {
- "vm.swappiness" => "1"
- }
- },
- :sched_wakeup => {
- :comment => "Tune scheduler",
- :parameters => {
- "kernel.sched_min_granularity_ns" => "10000000",
- "kernel.sched_wakeup_granularity_ns" => "15000000"
- }
- }
- },
- :tools => {
- :cron => {
- :load => {
- :nice => 19,
- :io_scheduling_class => "best-effort",
- :io_scheduling_priority => 7
- }
- }
- }
-)
-
-run_list(
- "recipe[tilecache]"
-)
description "Master role applied to trogdor"
default_attributes(
- :hardware => {
- :shm_size => "14g"
- },
:networking => {
:interfaces => {
:external_ipv4 => {
"block/md1/md/sync_speed_max" => "100000"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "10240 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :nginx => {
- :cache => {
- :proxy => {
- :max_size => "131072M"
- }
- }
- },
- :tilecache => {
- :tile_parent => "netherlands.render.openstreetmap.org"
}
)
run_list(
- "role[blix-nl]",
- "role[tilecache]"
+ "role[blix-nl]"
)
:anovak => { :status => :administrator }
}
},
- :hardware => {
- :shm_size => "36g"
- },
:location => "Pula, Croatia",
:munin => {
:allow => ["193.198.233.210"]
:gateway => "2001:b68:4cff:3::1"
}
}
- },
- :squid => {
- :version => 4,
- :cache_mem => "32768 MB",
- :cache_dir => [
- "rock /store/squid/rock-4096 20000 swap-timeout=200 slot-size=4096 max-size=3996",
- "rock /store/squid/rock-8192 25000 swap-timeout=200 slot-size=8192 min-size=3997 max-size=8092",
- "rock /store/squid/rock-16384 35000 swap-timeout=200 slot-size=16384 min-size=8093 max-size=16284",
- "rock /store/squid/rock-32768 45000 swap-timeout=200 slot-size=32768 min-size=16285 max-size=262144"
- ]
- },
- :tilecache => {
- :tile_parent => "pula.render.openstreetmap.org"
}
)
run_list(
- "role[carnet]",
- "role[tilecache]"
+ "role[carnet]"
)
+++ /dev/null
-require "serverspec"
-
-# Required by serverspec
-set :backend, :exec
-
-describe package("nginx") do
- it { should be_installed }
-end
-
-describe service("nginx") do
- it { should be_enabled }
- it { should be_running }
-end
-
-describe port(80) do
- it { should be_listening.with("tcp") }
-end
-
-describe port(443) do
- it { should be_listening.with("tcp") }
-end
-
-describe port(8050) do
- it { should be_listening.with("tcp") }
-end
+++ /dev/null
-require "serverspec"
-
-# Required by serverspec
-set :backend, :exec
-
-describe package("squid") do
- it { should be_installed }
-end
-
-describe service("squid") do
- it { should be_enabled }
- it { should be_running }
-end
-
-describe port(3128) do
- it { should be_listening.with("tcp") }
-end
-
-describe port(8080) do
- it { should be_listening.with("tcp") }
-end
projects / chef.git / commitdiff