From: Grant <git@firefishy.com>
Date: Thu, 3 Dec 2020 08:42:36 +0000 (+0000)
Subject: Merge pull request #362 from hbogner/master
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/a68381ff0b21ebe57d7cef919c34c4e31460bc45?hp=bd3a77ffbae2be8b346c4f1e0309790fdbb36a49

Merge pull request #362 from hbogner/master

Add croatian mirror to torrent webseed
---

diff --git a/Gemfile.lock b/Gemfile.lock
index dceed86bd..d4dbea51a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -4,18 +4,18 @@ GEM
     ast (2.4.1)
     bcrypt_pbkdf (1.0.1)
     builder (3.2.4)
-    chef-utils (16.6.14)
-    cookstyle (7.2.1)
-      rubocop (= 1.3.1)
+    chef-utils (16.7.61)
+    cookstyle (7.3.10)
+      rubocop (= 1.5.0)
     diff-lcs (1.4.4)
     docker-api (2.0.0)
       excon (>= 0.47.0)
       multi_json
     ed25519 (1.2.4)
-    erubi (1.9.0)
+    erubi (1.10.0)
     excon (0.78.0)
     ffi (1.13.1)
-    gssapi (1.3.0)
+    gssapi (1.3.1)
       ffi (>= 1.0.1)
     gyoku (1.3.1)
       builder (>= 2.1.2)
@@ -34,11 +34,11 @@ GEM
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
-    mixlib-install (3.12.3)
+    mixlib-install (3.12.5)
       mixlib-shellout
       mixlib-versioning
       thor
-    mixlib-shellout (3.1.6)
+    mixlib-shellout (3.2.2)
       chef-utils
     mixlib-versioning (1.2.12)
     multi_json (1.15.0)
@@ -49,13 +49,13 @@ GEM
       net-ssh (>= 4.0.0)
     net-telnet (0.1.1)
     nori (2.6.0)
-    parallel (1.20.0)
+    parallel (1.20.1)
     parser (2.7.2.0)
       ast (~> 2.4.1)
     pastel (0.8.0)
       tty-color (~> 0.5)
     rainbow (3.0.0)
-    regexp_parser (1.8.2)
+    regexp_parser (2.0.0)
     rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -73,16 +73,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    rubocop (1.3.1)
+    rubocop (1.5.0)
       parallel (~> 1.10)
       parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8)
+      regexp_parser (>= 2.0)
       rexml
-      rubocop-ast (>= 1.1.1)
+      rubocop-ast (>= 1.2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.1.1)
+    rubocop-ast (1.3.0)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.10.1)
     rubyntlm (0.6.2)
@@ -103,7 +103,7 @@ GEM
       unicode-display_width (~> 1.5)
       unicode_utils (~> 1.4)
     strings-ansi (0.2.0)
-    test-kitchen (2.7.2)
+    test-kitchen (2.8.0)
       bcrypt_pbkdf (~> 1.0)
       ed25519 (~> 1.2)
       license-acceptance (>= 1.0.11, < 3.0)
@@ -122,7 +122,7 @@ GEM
       pastel (~> 0.8)
       strings (~> 0.2.0)
       tty-cursor (~> 0.7)
-    tty-color (0.5.2)
+    tty-color (0.6.0)
     tty-cursor (0.7.1)
     tty-prompt (0.22.0)
       pastel (~> 0.8)
@@ -143,7 +143,7 @@ GEM
       logging (>= 1.6.1, < 3.0)
       nori (~> 2.0)
       rubyntlm (~> 0.6.0, >= 0.6.1)
-    winrm-elevated (1.2.2)
+    winrm-elevated (1.2.3)
       erubi (~> 1.8)
       winrm (~> 2.0)
       winrm-fs (~> 1.0)
diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb
index 2ac6e431f..8ac2886c2 100644
--- a/cookbooks/devices/templates/default/udev.rules.erb
+++ b/cookbooks/devices/templates/default/udev.rules.erb
@@ -85,6 +85,8 @@ SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1563
 SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1586", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096"
 # Ethernet controller: Intel Corporation Ethernet Controller X710 for 10GBASE-T
 SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x15ff", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096"
+# Ethernet controller: Intel Corporation Ethernet Connection I354 (rev 03)
+SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1f41", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096"
 # Ethernet controller: Intel Corporation Ethernet Connection X722 for 10GBASE-T
 SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x37d2", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096"
 
diff --git a/cookbooks/mediawiki/resources/site.rb b/cookbooks/mediawiki/resources/site.rb
index 2fc1ed115..598a7044e 100644
--- a/cookbooks/mediawiki/resources/site.rb
+++ b/cookbooks/mediawiki/resources/site.rb
@@ -458,6 +458,13 @@ action :create do
     update_site false
   end
 
+  mediawiki_extension "OSMCALWikiWidget" do
+    site new_resource.site
+    repository "https://github.com/thomersch/OSMCALWikiWidget.git"
+    tag "live"
+    update_site false
+  end
+
   mediawiki_extension "SimpleMap" do
     site new_resource.site
     template "mw-ext-SimpleMap.inc.php.erb"
diff --git a/cookbooks/munin/templates/default/munin.conf.erb b/cookbooks/munin/templates/default/munin.conf.erb
index fbedbb70b..3aa31b432 100644
--- a/cookbooks/munin/templates/default/munin.conf.erb
+++ b/cookbooks/munin/templates/default/munin.conf.erb
@@ -519,7 +519,6 @@ unknown_limit 144
     nginx_requests.graph_args --lower-limit 0
 <% @tilecaches.each do |tc| -%>
     nginx_requests.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-    nginx_requests.<%= tc[:name].tr("-", "_") %>.cdef <%= tc[:name].tr("-", "_") %>,8,*
     nginx_requests.<%= tc[:name].tr("-", "_") %>.draw AREASTACK
     nginx_requests.<%= tc[:name].tr("-", "_") %>.min 0
 <% end -%>
diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb
index e29fd7931..bf87600ba 100644
--- a/cookbooks/nominatim/attributes/default.rb
+++ b/cookbooks/nominatim/attributes/default.rb
@@ -13,10 +13,10 @@ default[:nominatim][:ui_repository] = "https://github.com/osm-search/nominatim-u
 default[:nominatim][:ui_revision] = "master"
 
 default[:nominatim][:fpm_pools] = {
-  :www => {
-    :port => 8000,
+  "nominatim.openstreetmap.org" => {
     :pm => "dynamic",
-    :max_children => 60
+    :max_children => 60,
+    :prometheus_port => 9253
   }
 }
 
diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb
index 75c357902..b5496a652 100644
--- a/cookbooks/nominatim/recipes/default.rb
+++ b/cookbooks/nominatim/recipes/default.rb
@@ -380,7 +380,7 @@ end
 end
 
 node[:nominatim][:fpm_pools].each do |name, data|
-  php_fpm name.to_s do
+  php_fpm name do
     port data[:port]
     pm data[:pm]
     pm_max_children data[:max_children]
@@ -388,6 +388,7 @@ node[:nominatim][:fpm_pools].each do |name, data|
     pm_min_spare_servers 10
     pm_max_spare_servers 20
     pm_max_requests 10000
+    prometheus_port data[:prometheus_port]
   end
 end
 
@@ -412,11 +413,13 @@ nginx_site "default" do
   action [:delete]
 end
 
+frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name)
+
 nginx_site "nominatim" do
   template "nginx.erb"
   directory build_directory
   variables :pools => node[:nominatim][:fpm_pools],
-            :frontends => search(:node, "recipes:web\\:\\:frontend"),
+            :frontends => frontends,
             :confdir => "#{basedir}/etc",
             :ui_directory => ui_directory
 end
@@ -454,9 +457,12 @@ end
 
 include_recipe "fail2ban"
 
+frontend_addresses = frontends.collect { |f| f.ipaddresses(:role => :external) }
+
 fail2ban_jail "nominatim_limit_req" do
   filter "nginx-limit-req"
   logpath "#{node[:nominatim][:logdir]}/nominatim.openstreetmap.org-error.log"
   ports [80, 443]
   maxretry 5
+  ignoreips frontend_addresses.flatten.sort
 end
diff --git a/cookbooks/nominatim/templates/default/nginx.erb b/cookbooks/nominatim/templates/default/nginx.erb
index d56d99c8a..a44e9382c 100644
--- a/cookbooks/nominatim/templates/default/nginx.erb
+++ b/cookbooks/nominatim/templates/default/nginx.erb
@@ -1,5 +1,5 @@
 upstream nominatim_service {
-  server 127.0.0.1:<%= @pools[:www][:port ]%>;
+  server unix:/run/php/nominatim.openstreetmap.org.sock;
 }
 
 map $uri $nominatim_script_name {
@@ -49,7 +49,7 @@ map $http_referer $missing_referer {
 geo $whitelisted {
     default 0;
 <% @frontends.each do |frontend| -%>
-<% frontend.ipaddresses(:role => :external) do |address| -%>
+<% frontend.ipaddresses(:role => :external).sort.each do |address| -%>
     <%= address %> 1;
 <% end -%>
 <% end -%>
@@ -86,9 +86,22 @@ map $blocked_user_agent $limit_tarpit {
     2 $binary_remote_addr;
 }
 
+map $missing_email$missing_referer$http_user_agent $generic_mozilla {
+    default 0;
+    ~^11Mozilla/4.0 1;
+    ~^11Mozilla/5.0 2;
+}
+
+map $whitelisted$generic_mozilla$uri $limit_reverse {
+    default "";
+    ~01/reverse.*  $binary_remote_addr;
+    ~02/reverse.*  $binary_remote_addr;
+}
+
 limit_req_zone $limit_www zone=www:50m rate=2r/s;
 limit_req_zone $limit_tarpit zone=tarpit:10m rate=1r/s;
 limit_req_zone $binary_remote_addr zone=blocked:10m rate=20r/m;
+limit_req_zone $limit_reverse zone=reverse:10m rate=10r/m;
 
 server {
     listen 80 default_server;
@@ -166,6 +179,7 @@ server {
 
         limit_req zone=www burst=10;
         limit_req zone=tarpit burst=2;
+        limit_req zone=reverse burst=5;
         limit_req_status 429;
         fastcgi_pass nominatim_service;
         include fastcgi_params;
@@ -188,6 +202,7 @@ server {
 
         limit_req zone=www burst=10;
         limit_req zone=tarpit burst=2;
+        limit_req zone=reverse burst=5;
         limit_req_status 429;
         fastcgi_pass    nominatim_service;
         include         fastcgi_params;
diff --git a/cookbooks/php/resources/fpm.rb b/cookbooks/php/resources/fpm.rb
index c461cf679..0178e64ce 100644
--- a/cookbooks/php/resources/fpm.rb
+++ b/cookbooks/php/resources/fpm.rb
@@ -45,7 +45,7 @@ action :create do
     owner "root"
     group "root"
     mode "644"
-    variables new_resource.to_hash
+    variables new_resource.to_hash.merge(:pool => new_resource.pool)
   end
 
   if new_resource.prometheus_port
diff --git a/cookbooks/planet/files/default/cgi/HEADER.cgi b/cookbooks/planet/files/default/cgi/HEADER.cgi
index 81dc70230..874c9c28d 100644
--- a/cookbooks/planet/files/default/cgi/HEADER.cgi
+++ b/cookbooks/planet/files/default/cgi/HEADER.cgi
@@ -69,7 +69,7 @@ print """
 The files found here are regularly-updated, complete copies of the OpenStreetMap.org
 database, and those published before the 12 September 2012 are distributed under a Creative Commons Attribution-ShareAlike 2.0 license, those published after are  Open Data Commons Open Database License 1.0 licensed. For more information, <a href="https://wiki.openstreetmap.org/wiki/Planet.osm">see the project wiki</a>.
 </p>
-<p><div class="alert"><strong>WARNING</strong> Download speeds are currently restricted to 4096 KB/s due to limited available capacity on our Internet connection. <a href="https://wiki.openstreetmap.org/wiki/Planet.osm#Planet.osm_mirrors">Please use a mirror if possible.</a></div></p>
+<p><div class="alert"><strong>WARNING</strong> Download speeds are currently restricted to 4096 KB/s due to limited available capacity on our Internet connection. <a href="https://wiki.openstreetmap.org/wiki/Planet.osm#BitTorrent">Please use torrents</a> or <a href="https://wiki.openstreetmap.org/wiki/Planet.osm#Planet.osm_mirrors">a mirror</a> if possible.</div></p>
 <table id="about">
   <tr>
     <th>
diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb
index 2a3e65cb2..165b9282a 100644
--- a/cookbooks/planet/recipes/replication.rb
+++ b/cookbooks/planet/recipes/replication.rb
@@ -137,6 +137,18 @@ directory "/store/planet/replication/test" do
   mode "755"
 end
 
+directory "/store/planet/replication/test/day" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+directory "/store/planet/replication/test/hour" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
 directory "/store/planet/replication/test/minute" do
   owner "planet"
   group "planet"
@@ -174,6 +186,18 @@ directory "/var/run/lock/changeset-replication/" do
   mode "750"
 end
 
+directory "/var/lib/replication" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+directory "/var/lib/replication/test" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
 template "/etc/replication/auth.conf" do
   source "replication.auth.erb"
   user "root"
@@ -223,6 +247,76 @@ systemd_timer "replication-minutely" do
   accuracy_sec 5
 end
 
+directory "/var/lib/replication/test/hour" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+template "/var/lib/replication/test/hour/configuration.txt" do
+  source "replication.config.erb"
+  owner "planet"
+  group "planet"
+  mode "644"
+  variables :base => "test/minute", :interval => 3600
+end
+
+link "/var/lib/replication/test/hour/data" do
+  to "/store/planet/replication/test/hour"
+end
+
+systemd_service "replication-hourly" do
+  description "Hourly replication"
+  user "planet"
+  exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour"
+  private_tmp true
+  private_devices true
+  protect_system "full"
+  protect_home true
+  restrict_address_families %w[AF_INET AF_INET6]
+  no_new_privileges true
+end
+
+systemd_timer "replication-hourly" do
+  description "Daily replication"
+  on_calendar "*-*-* *:02/15:00"
+end
+
+directory "/var/lib/replication/test/day" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+template "/var/lib/replication/test/day/configuration.txt" do
+  source "replication.config.erb"
+  owner "planet"
+  group "planet"
+  mode "644"
+  variables :base => "test/hour", :interval => 86400
+end
+
+link "/var/lib/replication/test/day/data" do
+  to "/store/planet/replication/test/day"
+end
+
+systemd_service "replication-daily" do
+  description "Daily replication"
+  user "planet"
+  exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day"
+  private_tmp true
+  private_devices true
+  protect_system "full"
+  protect_home true
+  restrict_address_families %w[AF_INET AF_INET6]
+  no_new_privileges true
+end
+
+systemd_timer "replication-daily" do
+  description "Daily replication"
+  on_calendar "*-*-* *:02/15:00"
+end
+
 template "/etc/replication/changesets.conf" do
   source "changesets.conf.erb"
   user "root"
@@ -239,12 +333,6 @@ template "/etc/replication/users-agreed.conf" do
   variables :password => db_passwords["planetdiff"]
 end
 
-directory "/var/lib/replication" do
-  owner "planet"
-  group "planet"
-  mode "755"
-end
-
 directory "/var/lib/replication/minute" do
   owner "planet"
   group "planet"
@@ -314,6 +402,14 @@ if node[:planet][:replication] == "enabled"
     action [:enable, :start]
   end
 
+  service "replication-hourly.timer" do
+    action [:enable, :start]
+  end
+
+  service "replication-daily.timer" do
+    action [:enable, :start]
+  end
+
   cron_d "replication-minutely" do
     user "planet"
     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
@@ -353,6 +449,14 @@ else
     action [:stop, :disable]
   end
 
+  service "replication-hourly.timer" do
+    action [:stop, :disable]
+  end
+
+  service "replication-daily.timer" do
+    action [:stop, :disable]
+  end
+
   cron_d "replication-minutely" do
     action :delete
   end
diff --git a/cookbooks/postgresql/attributes/default.rb b/cookbooks/postgresql/attributes/default.rb
index 2d9fc1079..c7eeecff8 100644
--- a/cookbooks/postgresql/attributes/default.rb
+++ b/cookbooks/postgresql/attributes/default.rb
@@ -2,6 +2,7 @@ default[:postgresql][:versions] = []
 default[:postgresql][:clusters] = {}
 default[:postgresql][:settings][:defaults][:port] = "5432"
 default[:postgresql][:settings][:defaults][:max_connections] = "100"
+default[:postgresql][:settings][:defaults][:ssl] = "true"
 default[:postgresql][:settings][:defaults][:shared_buffers] = "32MB"
 default[:postgresql][:settings][:defaults][:temp_buffers] = "8MB"
 default[:postgresql][:settings][:defaults][:work_mem] = "1MB"
diff --git a/cookbooks/postgresql/templates/default/postgresql.conf.erb b/cookbooks/postgresql/templates/default/postgresql.conf.erb
index 3c84ec994..e2892f68a 100644
--- a/cookbooks/postgresql/templates/default/postgresql.conf.erb
+++ b/cookbooks/postgresql/templates/default/postgresql.conf.erb
@@ -28,7 +28,7 @@ unix_socket_directory = '/var/run/postgresql'
 
 # - Security and Authentication -
 
-ssl = true
+ssl = <%= @settings[:ssl] || @defaults[:ssl] %>
 ssl_renegotiation_limit = 0
 
 #------------------------------------------------------------------------------
@@ -86,7 +86,9 @@ archive_command = '<%= @settings[:archive_command] || @defaults[:archive_command
 # - Sending Server(s) -
 
 max_wal_senders = <%= @settings[:max_wal_senders] || @defaults[:max_wal_senders] %>
+<% if @version.to_f >= 9.4 -%>
 max_replication_slots = <%= @settings[:max_replication_slots] || @defaults[:max_replication_slots] %>
+<% end -%>
 
 # - Standby Servers -
 
diff --git a/cookbooks/prometheus/templates/default/grafana.ini.erb b/cookbooks/prometheus/templates/default/grafana.ini.erb
index 08f1d5dfd..d3ea1ce65 100644
--- a/cookbooks/prometheus/templates/default/grafana.ini.erb
+++ b/cookbooks/prometheus/templates/default/grafana.ini.erb
@@ -15,3 +15,8 @@ host = localhost:25
 skip_verify = true
 from_address = admins@openstreetmap.org
 from_name = Prometheus
+
+[auth.anonymous]
+enabled = true
+org_name = OpenStreetMap
+org_role = Viewer
diff --git a/cookbooks/tile/files/default/html/favicon.ico b/cookbooks/tile/files/default/html/favicon.ico
index 27b042b5c..975e1cb0d 100644
Binary files a/cookbooks/tile/files/default/html/favicon.ico and b/cookbooks/tile/files/default/html/favicon.ico differ
diff --git a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
index 0ea85d755..338e0d51d 100644
--- a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
+++ b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
@@ -154,6 +154,14 @@ map $http_referer $denied_referer {
   '~^https?://[^.]*\.cellmapper\.net/'   1;
 }
 
+map $http_referer $censored_referer {
+  default                                 0; # Not denied
+  # Blocked on board instructions
+  '~^https?://schiebt-sie-ab\.de/'        1;
+  '~^https?://[^.]*\.schiebt-sie-ab\.de/' 1;
+}
+
+
 map $http_referer $osm_referer {
   default                                 '';    # False
   '~^https:\/\/www\.openstreetmap\.org\/' 'osm'; # True
@@ -405,6 +413,11 @@ server {
         return 418;
       }
 
+      if ($censored_referer) {
+        set $limit_rate 512;
+        return 451 "Unavailable at OSMF Board request";
+      }
+
       # Strip any ?query parameters from urls
       set $args '';
 
diff --git a/roles/dev.rb b/roles/dev.rb
index 565d4b70c..c192ac182 100644
--- a/roles/dev.rb
+++ b/roles/dev.rb
@@ -131,7 +131,7 @@ default_attributes(
     }
   },
   :postgresql => {
-    :versions => ["9.1", "12"],
+    :versions => ["12"],
     :settings => {
       :defaults => {
         :shared_buffers => "1GB",
@@ -140,9 +140,6 @@ default_attributes(
         :max_stack_depth => "4MB",
         :effective_cache_size => "4GB"
       },
-      "9.1" => {
-        :port => "5433"
-      },
       "12" => {
         :port => "5432",
         :wal_level => "logical",
diff --git a/roles/pummelzacken.rb b/roles/pummelzacken.rb
index b6d866984..05393514c 100644
--- a/roles/pummelzacken.rb
+++ b/roles/pummelzacken.rb
@@ -25,7 +25,7 @@ default_attributes(
     }
   },
   :postgresql => {
-    :versions => ["12"],
+    :versions => ["13"],
     :settings => {
       :defaults => {
         :listen_addresses => "10.0.0.20",
@@ -47,8 +47,8 @@ default_attributes(
   :nominatim => {
     :state => "standalone",
     :dbadmins => %w[lonvia tomh],
-    :dbcluster => "12/main",
-    :postgis => "2.5",
+    :dbcluster => "13/main",
+    :postgis => "3",
     :enable_backup => true,
     :flatnode_file => "/ssd/nominatim/nodes.store",
     :tablespaces => {