From: Grant <git@firefishy.com>
Date: Thu, 20 Aug 2020 11:19:06 +0000 (+0100)
Subject: Merge pull request #331 from Tigerfell/tiles
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/356a507d8627f062775ea315500c3f099588b5e3?hp=3736dc6e58fea1fc54fd42ffc872a1e82b7ba293

Merge pull request #331 from Tigerfell/tiles

Align wiki tile layer list with featured tile layers at osm.org
---

diff --git a/cookbooks/chef/attributes/default.rb b/cookbooks/chef/attributes/default.rb
index 640aad5cb..fc4a93c78 100644
--- a/cookbooks/chef/attributes/default.rb
+++ b/cookbooks/chef/attributes/default.rb
@@ -5,4 +5,4 @@ default[:apt][:sources] = node[:apt][:sources] | ["opscode"]
 default[:chef][:server][:version] = "12.17.33"
 
 # Set the default client version
-default[:chef][:client][:version] = "16.3.45"
+default[:chef][:client][:version] = "16.4.38"
diff --git a/cookbooks/civicrm/attributes/default.rb b/cookbooks/civicrm/attributes/default.rb
index 8b1fdc103..240c73468 100644
--- a/cookbooks/civicrm/attributes/default.rb
+++ b/cookbooks/civicrm/attributes/default.rb
@@ -1,4 +1,4 @@
-default[:civicrm][:version] = "5.27.3"
+default[:civicrm][:version] = "5.28.0"
 
 default[:civicrm][:extensions][:cividiscount][:name] = "org.civicrm.module.cividiscount"
 default[:civicrm][:extensions][:cividiscount][:repository] = "https://github.com/dlobo/org.civicrm.module.cividiscount.git"
@@ -31,7 +31,3 @@ default[:civicrm][:extensions][:username][:revision] = "master"
 default[:civicrm][:extensions][:donotsendreportemail][:name] = "org.civicrm.donotsendreportemail"
 default[:civicrm][:extensions][:donotsendreportemail][:repository] = "https://github.com/pradpnayak/org.civicrm.donotsendreportemail.git"
 default[:civicrm][:extensions][:donotsendreportemail][:revision] = "3b31c2e0c62183872c7ecd244395fb8dcfbd5dbb"
-
-default[:civicrm][:extensions][:donotsendreportemail][:name] = "nz.co.fuzion.omnipaymultiprocessor"
-default[:civicrm][:extensions][:donotsendreportemail][:repository] = "https://github.com/eileenmcnaughton/nz.co.fuzion.omnipaymultiprocessor.git"
-default[:civicrm][:extensions][:donotsendreportemail][:revision] = "3.10"
diff --git a/cookbooks/civicrm/recipes/default.rb b/cookbooks/civicrm/recipes/default.rb
index 936364026..b0717ef54 100644
--- a/cookbooks/civicrm/recipes/default.rb
+++ b/cookbooks/civicrm/recipes/default.rb
@@ -121,7 +121,7 @@ execute "/opt/civicrm-#{civicrm_version}/civicrm" do
   subscribes :run, "execute[#{cache_dir}/civicrm-#{civicrm_version}-l10n.tar.gz]", :immediately
 end
 
-directory "/srv/join.osmfoundation.org/wp-content/plugins/files" do
+directory "/srv/join.osmfoundation.org/wp-content/uploads" do
   owner "www-data"
   group "www-data"
   mode "755"
@@ -140,7 +140,6 @@ node[:civicrm][:extensions].each_value do |details|
     action :sync
     repository details[:repository]
     revision details[:revision]
-    depth 1
     user "wordpress"
     group "wordpress"
   end
@@ -157,10 +156,9 @@ settings = edit_file "#{civicrm_directory}/civicrm/templates/CRM/common/civicrm.
   line.gsub!(/%%dbHost%%/, "localhost")
   line.gsub!(/%%dbName%%/, "civicrm")
   line.gsub!(/%%crmRoot%%/, "#{civicrm_directory}/civicrm/")
-  line.gsub!(/%%templateCompileDir%%/, "/srv/join.osmfoundation.org/wp-content/plugins/files/civicrm/templates_c/")
+  line.gsub!(/%%templateCompileDir%%/, "/srv/join.osmfoundation.org/wp-content/uploads/civicrm/templates_c/")
   line.gsub!(/%%baseURL%%/, "http://join.osmfoundation.org/")
   line.gsub!(/%%siteKey%%/, site_key)
-  line.gsub!(%r{// *(.*'ext_repo_url'.*)$}, "\\1")
   line.gsub!(%r{// *define\('CIVICRM_CMSDIR', '/path/to/install/root/'\);}, "define('CIVICRM_CMSDIR', '/srv/join.osmfoundation.org');")
 
   line
diff --git a/cookbooks/forum/recipes/default.rb b/cookbooks/forum/recipes/default.rb
index 798c778d1..826310f73 100644
--- a/cookbooks/forum/recipes/default.rb
+++ b/cookbooks/forum/recipes/default.rb
@@ -65,8 +65,7 @@ end
 
 remote_file "#{cache_dir}/air3_v0.8.zip" do
   action :create_if_missing
-  source "https://grant.dev.openstreetmap.org/forum/air3_v0.8.zip" # Workaround OpenSSL chain of trust bug in chef https://fluxbb.org/resources/styles/air3/releases/0.8/air3_v0.8.zip
-  checksum "df547e3ac9596c1e6d9eedcb108559f84a28669763e24114ca6cdcbf118caf6c"
+  source "https://fluxbb.org/resources/styles/air3/releases/0.8/air3_v0.8.zip"
   owner "root"
   group "root"
   mode "644"
diff --git a/cookbooks/forum/templates/default/backup.cron.erb b/cookbooks/forum/templates/default/backup.cron.erb
index db4f0b412..517f71634 100644
--- a/cookbooks/forum/templates/default/backup.cron.erb
+++ b/cookbooks/forum/templates/default/backup.cron.erb
@@ -10,7 +10,7 @@ mkdir $T/forum-$D
 echo '[mysqldump]' > $T/mysqldump.opts
 echo 'user=forum' >> $T/mysqldump.opts
 echo 'password=<%= @passwords["database"] %>' >> $T/mysqldump.opts
-mysqldump --defaults-file=$T/mysqldump.opts --opt forum > $T/forum-$D/forum.sql
+mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces forum > $T/forum-$D/forum.sql
 ln -s /srv/forum.openstreetmap.org $T/forum-$D/www
 
 export RSYNC_RSH="ssh -ax"
diff --git a/cookbooks/geoipupdate/metadata.rb b/cookbooks/geoipupdate/metadata.rb
index 78d65c5fa..ccc09f626 100644
--- a/cookbooks/geoipupdate/metadata.rb
+++ b/cookbooks/geoipupdate/metadata.rb
@@ -7,3 +7,4 @@ description       "Installs and configures geoipupdate"
 version           "1.0.0"
 supports          "ubuntu"
 depends           "apt"
+depends           "systemd"
diff --git a/cookbooks/geoipupdate/recipes/default.rb b/cookbooks/geoipupdate/recipes/default.rb
index b27b439b6..27d6478ae 100644
--- a/cookbooks/geoipupdate/recipes/default.rb
+++ b/cookbooks/geoipupdate/recipes/default.rb
@@ -21,6 +21,19 @@ include_recipe "apt"
 
 license_keys = data_bag_item("geoipupdate", "license-keys")
 
+package "geoip-database" do
+  action :purge
+end
+
+package "geoip-database-contrib" do
+  action :purge
+end
+
+package "geoipupdate" do
+  action :purge
+  only_if { ::File.exist?("/etc/cron.d/geoipupdate") }
+end
+
 package "geoipupdate"
 
 template "/etc/GeoIP.conf" do
@@ -31,13 +44,39 @@ template "/etc/GeoIP.conf" do
   variables :license_keys => license_keys
 end
 
-execute "geoipdate" do
+execute "geoipupdate" do
   command "geoipupdate"
   user "root"
   group "root"
   not_if { ENV.key?("TEST_KITCHEN") || node[:geoipupdate][:editions].all? { |edition| ::File.exist?("/usr/share/GeoIP/#{edition}.mmdb") } }
 end
 
+systemd_service "geoipdate" do
+  action :delete
+end
+
+systemd_service "geoipupdate" do
+  description "Update GeoIP databases"
+  user "root"
+  exec_start "/usr/bin/geoipupdate"
+  private_tmp true
+  private_devices true
+  protect_system "strict"
+  protect_home true
+  read_write_paths "/usr/share/GeoIP"
+end
+
+systemd_timer "geoipupdate" do
+  description "Update GeoIP databases"
+  on_boot_sec "15m"
+  on_unit_active_sec "7d"
+  randomized_delay_sec "4h"
+end
+
+service "geoipupdate.timer" do
+  action [:enable, :start]
+end
+
 directory "/var/lib/GeoIP" do
   action :delete
   recursive true
diff --git a/cookbooks/imagery/templates/default/imagery.js.erb b/cookbooks/imagery/templates/default/imagery.js.erb
index 607e6e8fe..56ac172da 100644
--- a/cookbooks/imagery/templates/default/imagery.js.erb
+++ b/cookbooks/imagery/templates/default/imagery.js.erb
@@ -8,7 +8,7 @@ function createMap(divName) {
   var layers = L.control.layers(null, null, {collapsed:false}).addTo(map);
 
   // Add OpenStreetMap layer
-  layers.addBaseLayer(L.tileLayer("//tile-openstreetmap-org.global.ssl.fastly.net/{z}/{x}/{y}.png", {
+  layers.addBaseLayer(L.tileLayer("https://cdn-fastly-test.tile.openstreetmap.org/{z}/{x}/{y}.png", {
     attribution: "© <a target=\"_parent\" href=\"https://www.openstreetmap.org\">OpenStreetMap</a> and contributors, under an <a target=\"_parent\" href=\"https://www.openstreetmap.org/copyright\">open license</a>",
     maxZoom: 19
   }), "OpenStreetMap");
diff --git a/cookbooks/mediawiki/resources/site.rb b/cookbooks/mediawiki/resources/site.rb
index 472e8c3d8..424d4288a 100644
--- a/cookbooks/mediawiki/resources/site.rb
+++ b/cookbooks/mediawiki/resources/site.rb
@@ -44,6 +44,7 @@ property :recaptcha_public_key, :kind_of => String
 property :recaptcha_private_key, :kind_of => String
 property :extra_file_extensions, :kind_of => [String, Array], :default => []
 property :fpm_max_children, :kind_of => Integer, :default => 5
+property :fpm_request_terminate_timeout, :kind_of => Integer, :default => 300
 property :reload_apache, :kind_of => [TrueClass, FalseClass], :default => true
 
 action :create do
@@ -521,6 +522,7 @@ action :create do
 
   php_fpm new_resource.site do
     pm_max_children new_resource.fpm_max_children
+    request_terminate_timeout new_resource.fpm_request_terminate_timeout
     php_admin_values "open_basedir" => "#{site_directory}/:/usr/share/php/:/dev/null:/tmp/"
     php_values "memory_limit" => "500M",
                "max_execution_time" => "240",
diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
index cd6f3c6e4..cb4eb9ab9 100755
--- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
+++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
@@ -7,7 +7,7 @@ mkdir $T/wiki-<%= @name %>-$D
 echo '[mysqldump]' > $T/mysqldump.opts
 echo 'user=<%= @database_params[:username] %>' >> $T/mysqldump.opts
 echo 'password=<%= @database_params[:password] %>' >> $T/mysqldump.opts
-mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --single-transaction "<%= @database_params[:name] %>" | lz4 -9 > $T/wiki-<%= @name %>-$D/wiki.sql.lz4
+mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --single-transaction --no-tablespaces "<%= @database_params[:name] %>" | lz4 -9 > $T/wiki-<%= @name %>-$D/wiki.sql.lz4
 ln -s <%= @directory %>  $T/wiki-<%= @name %>-$D/www
 nice tar --create --dereference --directory=$T --warning=no-file-changed --exclude=wiki-<%= @name %>-$D/www/w/images/thumb --exclude=wiki-<%= @name %>-$D/www/w/.git --exclude=wiki-<%= @name %>-$D/www/w/extensions/*/.git wiki-<%= @name %>-$D | nice gzip --rsyncable -9 > $T/$B
 nice rsync --preallocate --fuzzy $T/$B backup::backup
diff --git a/cookbooks/munin/templates/default/backup.cron.erb b/cookbooks/munin/templates/default/backup.cron.erb
index a8d3a5671..13d4228ce 100644
--- a/cookbooks/munin/templates/default/backup.cron.erb
+++ b/cookbooks/munin/templates/default/backup.cron.erb
@@ -12,7 +12,7 @@ ln -s /var/lib/munin/*.storable $T/munin-$D
 
 export RSYNC_RSH="ssh -ax"
 
-nice tar --create --dereference --directory=$T munin-$D | nice gzip --rsyncable -9 > $T/$B
+nice tar --create --dereference --directory=$T --warning=no-file-removed munin-$D | nice gzip --rsyncable -9 > $T/$B
 nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/systemd/resources/service.rb b/cookbooks/systemd/resources/service.rb
index 9db776470..26cce2978 100644
--- a/cookbooks/systemd/resources/service.rb
+++ b/cookbooks/systemd/resources/service.rb
@@ -57,6 +57,9 @@ property :private_devices, [true, false]
 property :private_network, [true, false]
 property :protect_system, [TrueClass, FalseClass, String]
 property :protect_home, [TrueClass, FalseClass, String]
+property :read_write_paths, [String, Array]
+property :read_only_paths, [String, Array]
+property :inaccessible_paths, [String, Array]
 property :restrict_address_families, [String, Array]
 property :no_new_privileges, [true, false]
 property :tasks_max, Integer
diff --git a/cookbooks/systemd/templates/default/service.erb b/cookbooks/systemd/templates/default/service.erb
index 4828c96af..6d8a603cc 100644
--- a/cookbooks/systemd/templates/default/service.erb
+++ b/cookbooks/systemd/templates/default/service.erb
@@ -108,6 +108,15 @@ ProtectSystem=<%= @protect_system %>
 <% if @protect_home -%>
 ProtectHome=<%= @protect_home %>
 <% end -%>
+<% if @read_write_paths -%>
+ReadWritePaths=<%= Array(@read_write_paths).join(" ") %>
+<% end -%>
+<% if @read_only_paths -%>
+ReadOnlyPaths=<%= Array(@read_only_paths).join(" ") %>
+<% end -%>
+<% if @inaccessible_paths -%>
+InaccessiblePaths=<%= Array(@inaccessible_paths).join(" ") %>
+<% end -%>
 <% if @restrict_address_families -%>
 RestrictAddressFamilies=<%= Array(@restrict_address_families).join(" ") %>
 <% end -%>
diff --git a/cookbooks/tile/templates/default/apache.erb b/cookbooks/tile/templates/default/apache.erb
index 182b590e5..0e461b8d6 100644
--- a/cookbooks/tile/templates/default/apache.erb
+++ b/cookbooks/tile/templates/default/apache.erb
@@ -55,9 +55,9 @@
   RewriteRule ^/(\d+)/(\d+)/(\d+)\.png/dirty/?$   /default/$1/$2/$3.png/dirty  [PT,T=text/plain,L]
 
   # Historical Files redirect
-  RedirectPermanent /processed_p.tar.bz2 https://planet.openstreetmap.org/historical-shapefiles/processed_p.tar.bz2
-  RedirectPermanent /shoreline_300.tar.bz2 https://planet.openstreetmap.org/historical-shapefiles/shoreline_300.tar.bz2
-  RedirectPermanent /world_boundaries-spherical.tgz https://planet.openstreetmap.org/historical-shapefiles/world_boundaries-spherical.tgz
+  RedirectPermanent /processed_p.tar.bz2 https://planet.openstreetmap.org/historical-shapefiles/
+  RedirectPermanent /shoreline_300.tar.bz2 https://planet.openstreetmap.org/historical-shapefiles/
+  RedirectPermanent /world_boundaries-spherical.tgz https://planet.openstreetmap.org/historical-shapefiles/
 
   # Redirect ACME certificate challenges
   RedirectPermanent /.well-known/acme-challenge/ http://acme.openstreetmap.org/.well-known/acme-challenge/
diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb
index 6395fb1d5..f8ee4ca2e 100644
--- a/cookbooks/web/resources/rails_port.rb
+++ b/cookbooks/web/resources/rails_port.rb
@@ -80,7 +80,6 @@ action :create do
     ruby#{new_resource.ruby}-dev
     imagemagick
     nodejs
-    geoip-database
     tzdata
   ]
 
@@ -181,7 +180,6 @@ action :create do
 
     line.gsub!(/^( *)#geonames_username:.*$/, "\\1geonames_username: \"openstreetmap\"")
 
-    line.gsub!(/^( *)#geoip_database:.*$/, "\\1geoip_database: \"/usr/share/GeoIP/GeoIPv6.dat\"")
     line.gsub!(/^( *)#maxmind_database:.*$/, "\\1maxmind_database: \"/usr/share/GeoIP/GeoLite2-Country.mmdb\"")
 
     if new_resource.gpx_dir
@@ -327,7 +325,6 @@ action :create do
     "support_email" => "support@openstreetmap.org",
     "email_return_path" => "bounces@openstreetmap.org",
     "geonames_username" => "openstreetmap",
-    "geoip_database" => "/usr/share/GeoIP/GeoIPv6.dat",
     "maxmind_database" => "/usr/share/GeoIP/GeoLite2-Country.mmdb"
   )
 
diff --git a/roles/db-master.rb b/roles/db-master.rb
index 19f9d55d9..d65791e15 100644
--- a/roles/db-master.rb
+++ b/roles/db-master.rb
@@ -10,6 +10,7 @@ default_attributes(
         :archive_command => "/usr/local/bin/openstreetmap-wal-e --terse wal-push %p",
         :max_wal_senders => "3",
         :late_authentication_rules => [
+          { :database => "replication", :user => "replication", :address => "10.0.48.49/32" },
           { :database => "replication", :user => "replication", :address => "10.0.48.50/32" },
           { :database => "replication", :user => "replication", :address => "10.0.48.5/32" },
           { :database => "replication", :user => "replication", :address => "10.0.0.10/32" },
diff --git a/roles/snap-01.rb b/roles/snap-01.rb
index 00b8d4b2a..c79f14145 100644
--- a/roles/snap-01.rb
+++ b/roles/snap-01.rb
@@ -14,9 +14,31 @@ default_attributes(
         }
       }
     }
+  },
+  :postgresql => {
+    :settings => {
+      :defaults => {
+        :shared_buffers => "128GB",
+        :work_mem => "128MB",
+        :maintenance_work_mem => "2GB",
+        :effective_cache_size => "360GB",
+        :effective_io_concurrency => "256",
+        :random_page_cost => "1.1"
+      }
+    }
+  },
+  :sysctl => {
+    :postgres => {
+      :comment => "Increase shared memory for postgres",
+      :parameters => {
+        "kernel.shmmax" => 132 * 1024 * 1024 * 1024,
+        "kernel.shmall" => 132 * 1024 * 1024 * 1024 / 4096
+      }
+    }
   }
 )
 
 run_list(
-  "role[equinix]"
+  "role[equinix]",
+  "role[db-slave]"
 )
diff --git a/roles/wiki.rb b/roles/wiki.rb
index acd0cef1c..be9510f70 100644
--- a/roles/wiki.rb
+++ b/roles/wiki.rb
@@ -8,7 +8,14 @@ default_attributes(
     }
   },
   :apache => {
-    :timeout => 30
+    :mpm => "event",
+    :timeout => 30,
+    :event => {
+      :server_limit => 32,
+      :max_request_workers => 800,
+      :threads_per_child => 50,
+      :max_connections_per_child => 10000
+    }
   },
   :elasticsearch => {
     :version => "5.x",