From 6ef9a5cf22dbff4124c3c02844825ac0917b48da Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 7 Mar 2025 15:01:41 +0000 Subject: [PATCH 01/16] Improve routing policy rules for Dublin --- roles/equinix-dub-public.rb | 17 ++++++----------- roles/equinix-dub.rb | 7 ++++++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/equinix-dub-public.rb b/roles/equinix-dub-public.rb index e00975a76..4aea8ef63 100644 --- a/roles/equinix-dub-public.rb +++ b/roles/equinix-dub-public.rb @@ -12,19 +12,11 @@ default_attributes( :source_route_table => 101, :inet => { :prefix => "27", - :gateway => "184.104.226.97", - :rules => [ - { :to => "10.0.0.0/8", :table => "main", :priority => 50 }, - { :to => "172.16.0.0/12", :table => "main", :priority => 50 }, - { :to => "192.168.0.0/16", :table => "main", :priority => 50 } - ] + :gateway => "184.104.226.97" }, :inet6 => { :prefix => 64, - :gateway => "2001:470:1:b3b::1", - :rules => [ - { :to => "2600:9000::/28", :table => 150, :priority => 100 } - ] + :gateway => "2001:470:1:b3b::1" } }, :equinix => { @@ -39,7 +31,10 @@ default_attributes( }, :inet6 => { :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :gateway => "2001:4d78:fe03:1c::1", + :rules => [ + { :to => "2600:9000::/28", :table => 203, :priority => 100 } + ] } } } diff --git a/roles/equinix-dub.rb b/roles/equinix-dub.rb index 716f0ff7c..741c09ab7 100644 --- a/roles/equinix-dub.rb +++ b/roles/equinix-dub.rb @@ -22,7 +22,12 @@ default_attributes( :gateway => "10.0.64.2", :routes => { "10.0.0.0/8" => { :via => "10.0.64.2" } - } + }, + :rules => [ + { :to => "10.0.0.0/8", :table => "main", :priority => 50 }, + { :to => "172.16.0.0/12", :table => "main", :priority => 50 }, + { :to => "192.168.0.0/16", :table => "main", :priority => 50 } + ] }, :bond => { :mode => "802.3ad", -- 2.39.5 From d54777514c0bb1fe1091c584aa4583041d42e1cf Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 7 Mar 2025 15:15:23 +0000 Subject: [PATCH 02/16] Improve blackholing of AWS routes on he.net --- cookbooks/networking/templates/default/network.erb | 6 ++++++ roles/equinix-ams-public.rb | 5 ++++- roles/equinix-dub-public.rb | 10 +++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cookbooks/networking/templates/default/network.erb b/cookbooks/networking/templates/default/network.erb index 7f38de3c5..f837c65a5 100644 --- a/cookbooks/networking/templates/default/network.erb +++ b/cookbooks/networking/templates/default/network.erb @@ -91,6 +91,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> @@ -107,6 +110,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> diff --git a/roles/equinix-ams-public.rb b/roles/equinix-ams-public.rb index def6bc25f..a67b1bfcc 100644 --- a/roles/equinix-ams-public.rb +++ b/roles/equinix-ams-public.rb @@ -16,7 +16,10 @@ default_attributes( }, :inet6 => { :prefix => "64", - :gateway => "2001:470:1:fa1::1" + :gateway => "2001:470:1:fa1::1", + :routes => { + "2600:9000::/28" => { :table => 3, :type => "unreachable" } + } } }, :equinix => { diff --git a/roles/equinix-dub-public.rb b/roles/equinix-dub-public.rb index 4aea8ef63..3cac11b43 100644 --- a/roles/equinix-dub-public.rb +++ b/roles/equinix-dub-public.rb @@ -16,7 +16,10 @@ default_attributes( }, :inet6 => { :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :gateway => "2001:470:1:b3b::1", + :routes => { + "2600:9000::/28" => { :table => 101, :type => "unreachable" } + } } }, :equinix => { @@ -31,10 +34,7 @@ default_attributes( }, :inet6 => { :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1", - :rules => [ - { :to => "2600:9000::/28", :table => 203, :priority => 100 } - ] + :gateway => "2001:4d78:fe03:1c::1" } } } -- 2.39.5 From 7d537eb5d7627524e24c7d3f404b6472e0130657 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 7 Mar 2025 16:59:00 +0000 Subject: [PATCH 03/16] Move OSUOSL hosts to use interface inheritance for network configuration --- roles/osuosl.rb | 4 +++- roles/piasa.rb | 2 -- roles/stormfly-03.rb | 2 -- roles/stormfly-04.rb | 2 -- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/roles/osuosl.rb b/roles/osuosl.rb index 99a3a992f..83fd46a72 100644 --- a/roles/osuosl.rb +++ b/roles/osuosl.rb @@ -11,8 +11,10 @@ default_attributes( :location => "Corvallis, Oregon", :timezone => "US/Pacific", :networking => { - :roles => { + :interfaces => { :external => { + :interface => "bond0", + :role => :external, :inet => { :prefix => "28", :gateway => "140.211.167.97" diff --git a/roles/piasa.rb b/roles/piasa.rb index b1e775ad4..085fdc66e 100644 --- a/roles/piasa.rb +++ b/roles/piasa.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.101" }, diff --git a/roles/stormfly-03.rb b/roles/stormfly-03.rb index d81b06c9f..374d2ab90 100644 --- a/roles/stormfly-03.rb +++ b/roles/stormfly-03.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.99" }, diff --git a/roles/stormfly-04.rb b/roles/stormfly-04.rb index 3a7d1a7c3..bcc11518a 100644 --- a/roles/stormfly-04.rb +++ b/roles/stormfly-04.rb @@ -8,8 +8,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.100" }, -- 2.39.5 From 5d3548bfe8bfd95ccc9441408c51a96c48d6c211 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 7 Mar 2025 17:04:25 +0000 Subject: [PATCH 04/16] Drop last remnants of old bytemark networking --- roles/bytemark.rb | 25 +------------------------ roles/shenron.rb | 13 +++++-------- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/roles/bytemark.rb b/roles/bytemark.rb index 8f23bd5d0..109a17208 100644 --- a/roles/bytemark.rb +++ b/roles/bytemark.rb @@ -3,30 +3,7 @@ description "Role applied to all servers at Bytemark" default_attributes( :hosted_by => "Bytemark", - :location => "York, England", - :networking => { - :roles => { - :internal => { - :inet => { - :prefix => "20", - :gateway => "10.0.32.20", - :routes => { - "10.0.0.0/8" => { :via => "10.0.32.20" } - } - } - }, - :external => { - :inet => { - :prefix => "28", - :gateway => "89.16.162.17" - }, - :inet6 => { - :prefix => "64", - :gateway => "fe80::1" - } - } - } - } + :location => "York, England" ) override_attributes( diff --git a/roles/shenron.rb b/roles/shenron.rb index f91677226..7f9f36a1c 100644 --- a/roles/shenron.rb +++ b/roles/shenron.rb @@ -19,14 +19,6 @@ default_attributes( "it87" ] }, - :prometheus => { - :metrics => { - :exim_queue_limit => { :metric => 250 } - } - } -) - -override_attributes( :networking => { :dnssec => "false", :interfaces => { @@ -45,6 +37,11 @@ override_attributes( } } } + }, + :prometheus => { + :metrics => { + :exim_queue_limit => { :metric => 250 } + } } ) -- 2.39.5 From b977046c8a723fc935d376ec18a1c708f7c26339 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 7 Mar 2025 17:17:55 +0000 Subject: [PATCH 05/16] Drop last remnants of role based networking --- cookbooks/networking/recipes/default.rb | 35 ++++++------------------- roles/base.rb | 4 --- roles/firefishynet.rb | 2 +- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/cookbooks/networking/recipes/default.rb b/cookbooks/networking/recipes/default.rb index 4ab91c053..831878ab4 100644 --- a/cookbooks/networking/recipes/default.rb +++ b/cookbooks/networking/recipes/default.rb @@ -59,37 +59,18 @@ interfaces = node[:networking][:interfaces].collect do |name, interface| [interface[:interface], name] end.to_h -node[:networking][:interfaces].each do |name, interface| - if interface[:interface] =~ /^(.*)\.(\d+)$/ - vlan_interface = Regexp.last_match(1) - vlan_id = Regexp.last_match(2) - - parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" - - node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface - node.default_unless[:networking][:interfaces][parent][:vlans] = [] - - node.default[:networking][:interfaces][parent][:vlans] << vlan_id - end +node[:networking][:interfaces].each do |_, interface| + next unless interface[:interface] =~ /^(.*)\.(\d+)$/ - next unless interface[:role] && (role = node[:networking][:roles][interface[:role]]) + vlan_interface = Regexp.last_match(1) + vlan_id = Regexp.last_match(2) - if interface[:inet] && role[:inet] - node.default_unless[:networking][:interfaces][name][:inet][:prefix] = role[:inet][:prefix] - node.default_unless[:networking][:interfaces][name][:inet][:gateway] = role[:inet][:gateway] - node.default_unless[:networking][:interfaces][name][:inet][:routes] = role[:inet][:routes] - node.default_unless[:networking][:interfaces][name][:inet][:rules] = role[:inet][:rules] - end + parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" - if interface[:inet6] && role[:inet6] - node.default_unless[:networking][:interfaces][name][:inet6][:prefix] = role[:inet6][:prefix] - node.default_unless[:networking][:interfaces][name][:inet6][:gateway] = role[:inet6][:gateway] - node.default_unless[:networking][:interfaces][name][:inet6][:routes] = role[:inet6][:routes] - node.default_unless[:networking][:interfaces][name][:inet6][:rules] = role[:inet6][:rules] - end + node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface + node.default_unless[:networking][:interfaces][parent][:vlans] = [] - node.default_unless[:networking][:interfaces][name][:metric] = role[:metric] - node.default_unless[:networking][:interfaces][name][:zone] = role[:zone] + node.default[:networking][:interfaces][parent][:vlans] << vlan_id end node[:networking][:interfaces].each do |_, interface| diff --git a/roles/base.rb b/roles/base.rb index 2a4f732a5..ee9b003c6 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -11,10 +11,6 @@ default_attributes( } }, :networking => { - :roles => { - :internal => { :metric => 200 }, - :external => { :metric => 100 } - }, :search => ["openstreetmap.org"] }, :prometheus => { diff --git a/roles/firefishynet.rb b/roles/firefishynet.rb index 150f62d6f..bb4224f70 100644 --- a/roles/firefishynet.rb +++ b/roles/firefishynet.rb @@ -3,7 +3,7 @@ description "Role applied to all servers at Firefishy" default_attributes( :networking => { - :roles => { + :interfaces => { :internal => { :inet => { :prefix => "24", -- 2.39.5 From 31bde6183f16c517ef2ce8f713642d0062e65000 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 7 Mar 2025 17:26:26 +0000 Subject: [PATCH 06/16] Drop last traces of old shorewall firewall zones --- cookbooks/dhcpd/recipes/default.rb | 3 +-- cookbooks/networking/attributes/default.rb | 1 - cookbooks/openssh/recipes/default.rb | 2 +- roles/equinix-ams-public.rb | 2 -- roles/equinix-dub-public.rb | 2 -- roles/ucl-public.rb | 1 - 6 files changed, 2 insertions(+), 9 deletions(-) diff --git a/cookbooks/dhcpd/recipes/default.rb b/cookbooks/dhcpd/recipes/default.rb index 0f3c2be9c..685d08c3b 100644 --- a/cookbooks/dhcpd/recipes/default.rb +++ b/cookbooks/dhcpd/recipes/default.rb @@ -53,8 +53,7 @@ remote_file "/srv/tftp/netboot.xyz.kpxe" do mode "644" end -zone = node.interfaces(:role => :external).first[:zone] -domain = "#{zone}.openstreetmap.org" +domain = node[:networking][:search].first template "/etc/dhcp/dhcpd.conf" do source "dhcpd.conf.erb" diff --git a/cookbooks/networking/attributes/default.rb b/cookbooks/networking/attributes/default.rb index 9832ce8f3..7ff93c365 100644 --- a/cookbooks/networking/attributes/default.rb +++ b/cookbooks/networking/attributes/default.rb @@ -6,7 +6,6 @@ default[:networking][:firewall][:outgoing] = [] default[:networking][:firewall][:http_rate_limit] = nil default[:networking][:firewall][:http_connection_limit] = nil default[:networking][:firewall][:allowlist] = [] -default[:networking][:roles] = {} default[:networking][:interfaces] = {} default[:networking][:nameservers] = %w[8.8.8.8 8.8.4.4 2001:4860:4860::8888 2001:4860:4860::8844] default[:networking][:search] = [] diff --git a/cookbooks/openssh/recipes/default.rb b/cookbooks/openssh/recipes/default.rb index 8b57aaaef..98e2fa9f2 100644 --- a/cookbooks/openssh/recipes/default.rb +++ b/cookbooks/openssh/recipes/default.rb @@ -43,7 +43,7 @@ hosts = search(:node, "networking:interfaces").sort_by { |n| n[:hostname] }.coll names = [name] unless node.interfaces(:role => :internal).empty? - names.unshift("#{name}.#{node[:networking][:roles][:external][:zone]}.openstreetmap.org") + names.unshift("#{name}.#{node[:networking][:search].first}") end unless node.interfaces(:role => :external).empty? diff --git a/roles/equinix-ams-public.rb b/roles/equinix-ams-public.rb index a67b1bfcc..b89ae21bb 100644 --- a/roles/equinix-ams-public.rb +++ b/roles/equinix-ams-public.rb @@ -7,7 +7,6 @@ default_attributes( :henet => { :interface => "bond0.3", :role => :external, - :zone => "ams", :metric => 150, :source_route_table => 3, :inet => { @@ -25,7 +24,6 @@ default_attributes( :equinix => { :interface => "bond0.103", :role => :external, - :zone => "ams", :metric => 100, :source_route_table => 103, :inet => { diff --git a/roles/equinix-dub-public.rb b/roles/equinix-dub-public.rb index 3cac11b43..343dfb4f9 100644 --- a/roles/equinix-dub-public.rb +++ b/roles/equinix-dub-public.rb @@ -7,7 +7,6 @@ default_attributes( :henet => { :interface => "bond0.101", :role => :external, - :zone => "dub", :metric => 150, :source_route_table => 101, :inet => { @@ -25,7 +24,6 @@ default_attributes( :equinix => { :interface => "bond0.203", :role => :external, - :zone => "dub", :metric => 100, :source_route_table => 203, :inet => { diff --git a/roles/ucl-public.rb b/roles/ucl-public.rb index 774630863..7b7c6188a 100644 --- a/roles/ucl-public.rb +++ b/roles/ucl-public.rb @@ -7,7 +7,6 @@ default_attributes( :external => { :role => :external, :metric => 100, - :zone => "ucl", :inet => { :prefix => "24", :gateway => "193.60.236.254" -- 2.39.5 From d5fde42b2c21b8559d5dc5e6c31a4750e2069177 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Sun, 9 Mar 2025 14:29:41 +0000 Subject: [PATCH 07/16] Disable DNSSEC validation in systemd-resolved --- cookbooks/networking/attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/networking/attributes/default.rb b/cookbooks/networking/attributes/default.rb index 7ff93c365..06511d462 100644 --- a/cookbooks/networking/attributes/default.rb +++ b/cookbooks/networking/attributes/default.rb @@ -9,7 +9,7 @@ default[:networking][:firewall][:allowlist] = [] default[:networking][:interfaces] = {} default[:networking][:nameservers] = %w[8.8.8.8 8.8.4.4 2001:4860:4860::8888 2001:4860:4860::8844] default[:networking][:search] = [] -default[:networking][:dnssec] = "allow-downgrade" +default[:networking][:dnssec] = "false" default[:networking][:hostname] = node.name default[:networking][:wireguard][:enabled] = true default[:networking][:wireguard][:keepalive] = 180 -- 2.39.5 From c37bde0b14664542dc192faa9a051304e4339985 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Sun, 9 Mar 2025 14:43:08 +0000 Subject: [PATCH 08/16] Increase mysql connection limit for the wiki --- roles/wiki.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/wiki.rb b/roles/wiki.rb index 19a31e3db..652726346 100644 --- a/roles/wiki.rb +++ b/roles/wiki.rb @@ -59,7 +59,7 @@ default_attributes( :mysqld => { :innodb_buffer_pool_size => "4G", :key_buffer_size => "64M", - :max_connections => "500", + :max_connections => "800", :sort_buffer_size => "8M", :tmp_table_size => "128M" } -- 2.39.5 From 82cb6ba5aad6a32e84ef77ddff1d66ef3aed1438 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Sun, 9 Mar 2025 19:12:05 +0000 Subject: [PATCH 09/16] dns: Update dnscontrol to 4.17.0 --- cookbooks/dns/recipes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index 3a19d8a91..0ec59c8b3 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -42,7 +42,7 @@ package %w[ cache_dir = Chef::Config[:file_cache_path] -dnscontrol_version = "4.15.5" +dnscontrol_version = "4.17.0" dnscontrol_arch = if arm? "arm64" -- 2.39.5 From 798b617b375179b76e65920fed450a5ab2325f37 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 9 Mar 2025 17:27:20 -0700 Subject: [PATCH 10/16] Re-render vector tiles by layer This should reduce the amount of work that is caused by updates. --- cookbooks/vectortile/recipes/default.rb | 4 +++- cookbooks/vectortile/templates/default/tiles-rerender.erb | 7 +++++-- .../vectortile/templates/default/vector-update-tile.erb | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb index feda712a0..0f4041565 100644 --- a/cookbooks/vectortile/recipes/default.rb +++ b/cookbooks/vectortile/recipes/default.rb @@ -253,12 +253,14 @@ template "/usr/local/bin/vector-update" do variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender" end +rerender_layers = %w[addresses boundaries bridges buildings land pois public_transport sites street_polygons streets water_lines_labels water_lines water_polygons].join(" ") + template "/usr/local/bin/tiles-rerender" do source "tiles-rerender.erb" owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4 + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4, :layers => "#{rerender_layers}" end systemd_service "replicate" do diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb index 6c01b27f6..d1ca4bdab 100644 --- a/cookbooks/vectortile/templates/default/tiles-rerender.erb +++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb @@ -2,9 +2,12 @@ set -eu cd "<%= @expiry_dir %>" - +# Only expire z10+ tiles +rm -f z?-*.txt wc -l z*.txt -cat z*.txt | <%= @tilekiln_bin %> generate tiles \ +(for layer in <%= @layers %>; do + find . -type f -name "z*-${layer}.txt" -execdir sed -e "s/\$/,${layer}/" {} \; +done) | <%= @tilekiln_bin %> generate layers \ --source-dbname "<%= @source_database %>" \ --storage-dbname "<%= @storage_database %>" \ --num-threads "<%= node[:vectortile][:replication][:threads] %>" \ diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb index ba708c9b8..a356cbb10 100644 --- a/cookbooks/vectortile/templates/default/vector-update-tile.erb +++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb @@ -8,6 +8,8 @@ set -eu export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' cd "<%= @expiry_dir %>" +rm -f z*.txt + osm2pgsql-replication update \ -d "<%= @source_database %>" \ --max-diff-size "<%= @diff_size %>" \ -- 2.39.5 From 35f2a545fa4791c849a3338115f0a1596242a727 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Tue, 18 Mar 2025 19:42:06 +0000 Subject: [PATCH 11/16] Add support for i18n-js 4.x --- cookbooks/web/resources/rails_port.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb index 681690452..ab589f6f3 100644 --- a/cookbooks/web/resources/rails_port.rb +++ b/cookbooks/web/resources/rails_port.rb @@ -453,7 +453,7 @@ action :create do only_if { new_resource.build_assets } end - bundle_exec "#{rails_directory}/app/assets/javascripts/i18n" do + bundle_exec "#{rails_directory}/config/i18n-js.yml" do action :nothing directory rails_directory command "rails i18n:js:export" @@ -463,7 +463,18 @@ action :create do user new_resource.user group new_resource.group subscribes :run, "git[#{rails_directory}]" - only_if { new_resource.build_assets } + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n-js.yml") } + end + + bundle_exec "#{rails_directory}/config/i18n.yml" do + action :nothing + directory rails_directory + command "i18n export" + environment "HOME" => rails_directory + user new_resource.user + group new_resource.group + subscribes :run, "git[#{rails_directory}]" + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n.yml") } end bundle_exec "#{rails_directory}/public/assets" do -- 2.39.5 From 38f4193b937e0999dbcf81d40af424e1eb9f35e6 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 20 Mar 2025 12:41:24 +0000 Subject: [PATCH 12/16] Fix i18n dependencies in rails asset rebuilds --- cookbooks/web/resources/rails_port.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb index ab589f6f3..cb71e73a5 100644 --- a/cookbooks/web/resources/rails_port.rb +++ b/cookbooks/web/resources/rails_port.rb @@ -491,7 +491,8 @@ action :create do subscribes :run, "file[#{rails_directory}/config/settings.local.yml]" subscribes :run, "file[#{rails_directory}/config/storage.yml]" subscribes :run, "bundle_exec[#{rails_directory}/package.json]" - subscribes :run, "bundle_exec[#{rails_directory}/app/assets/javascripts/i18n]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n-js.yml]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n.yml]" only_if { new_resource.build_assets } end -- 2.39.5 From d624653de9fbf581faa2e33dc914eff4551a483e Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 20 Mar 2025 22:12:58 +0000 Subject: [PATCH 13/16] Switch gps-tile tests to run on Debian --- .github/workflows/test-kitchen.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index e1c8df555..80ebb7cba 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -135,6 +135,8 @@ jobs: suite: git-server - os: debian-12 suite: git-web + - os: debian-12 + suite: gps-tile - os: debian-12 suite: imagery-tiler - os: debian-12 @@ -164,6 +166,8 @@ jobs: os: ubuntu-2204 - suite: git-web os: ubuntu-2204 + - suite: gps-tile + os: ubuntu-2204 - suite: mailman os: ubuntu-2204 - suite: letsencrypt -- 2.39.5 From f7d424073074cd02e19dc6fe3670c6a850020ec9 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 20 Mar 2025 22:42:43 +0000 Subject: [PATCH 14/16] Make wiki installation configurable --- cookbooks/wiki/attributes/default.rb | 9 +++-- cookbooks/wiki/recipes/default.rb | 51 ++++++++++++++-------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cookbooks/wiki/attributes/default.rb b/cookbooks/wiki/attributes/default.rb index 0cde6a313..e843d9a95 100644 --- a/cookbooks/wiki/attributes/default.rb +++ b/cookbooks/wiki/attributes/default.rb @@ -1,2 +1,7 @@ -# Force apache to listen only on localhost -# default[:apache][:listen_address] = "127.0.0.1" +default[:wiki][:site_name] = "wiki.openstreetmap.org" +default[:wiki][:site_aliases] = [ + "wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", + "osm.wiki", "www.osm.wiki", "wiki.osm.wiki" +] +default[:wiki][:site_notice] = nil +default[:wiki][:site_readonly] = nil diff --git a/cookbooks/wiki/recipes/default.rb b/cookbooks/wiki/recipes/default.rb index 74e96d2cc..6e58b3703 100644 --- a/cookbooks/wiki/recipes/default.rb +++ b/cookbooks/wiki/recipes/default.rb @@ -19,6 +19,8 @@ include_recipe "mediawiki" +site_name = node[:wiki][:site_name] + passwords = data_bag_item("wiki", "passwords") package "lua5.1" # newer versions do not work with Scribuntu! @@ -27,9 +29,8 @@ apache_site "default" do action [:disable] end -mediawiki_site "wiki.openstreetmap.org" do - aliases ["wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", - "osm.wiki", "www.osm.wiki", "wiki.osm.wiki"] +mediawiki_site site_name do + aliases node[:wiki][:site_aliases] fpm_max_children 200 fpm_start_servers 25 @@ -54,54 +55,54 @@ mediawiki_site "wiki.openstreetmap.org" do hcaptcha_public_key "b67a410b-955e-4049-b432-f9c00e0202c0" hcaptcha_private_key passwords["hcaptcha"] - # site_notice "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." - # site_readonly "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." + site_notice node[:wiki][:site_notice] + site_readonly node[:wiki][:site_readonly] end mediawiki_extension "CodeEditor" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "CodeMirror" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "Scribunto" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Scribunto.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Wikibase" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Wikibase.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "OsmWikibase" do - site "wiki.openstreetmap.org" + site site_name repository "https://github.com/nyurik/OsmWikibase.git" reference "master" end mediawiki_extension "Echo" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Echo.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Thanks" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Thanks.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "TimedMediaHandler" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "MultiMaps" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-MultiMaps.inc.php.erb" template_cookbook "wiki" variables :thunderforest_key => passwords["thunderforest"] @@ -109,37 +110,37 @@ mediawiki_extension "MultiMaps" do end mediawiki_extension "JsonConfig" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-JsonConfig.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Kartographer" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Kartographer.inc.php.erb" template_cookbook "wiki" end -cookbook_file "/srv/wiki.openstreetmap.org/osm_logo_wiki.png" do +cookbook_file "/srv/#{site_name}/osm_logo_wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -template "/srv/wiki.openstreetmap.org/robots.txt" do +template "/srv/#{site_name}/robots.txt" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" source "robots.txt.erb" end -cookbook_file "/srv/wiki.openstreetmap.org/favicon.ico" do +cookbook_file "/srv/#{site_name}/favicon.ico" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -directory "/srv/wiki.openstreetmap.org/dump" do +directory "/srv/#{site_name}/dump" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "0775" @@ -149,13 +150,13 @@ systemd_service "wiki-dump" do description "Wiki dump" type "oneshot" exec_start "/usr/bin/php w/maintenance/dumpBackup.php --full --quiet --output=gzip:dump/dump.xml.gz" - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" nice 19 sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-dump" do @@ -173,14 +174,14 @@ systemd_service "wiki-rdf-dump" do exec_start [ "/usr/bin/php w/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki wiki --format ttl --flavor full-dump --entity-type item --entity-type property --no-cache --output /tmp/wikibase-rdf.ttl", "/bin/gzip -9 /tmp/wikibase-rdf.ttl", - "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/wiki.openstreetmap.org/dump/wikibase-rdf.ttl.gz" + "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/#{site_name}/dump/wikibase-rdf.ttl.gz" ] - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-rdf-dump" do -- 2.39.5 From ef14381f4b0dbd1b0a9cbfa971b5f6543e450871 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 20 Mar 2025 22:43:26 +0000 Subject: [PATCH 15/16] Test wiki cookbook on Debian 12 --- .github/workflows/test-kitchen.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index 80ebb7cba..143ed1216 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -155,6 +155,8 @@ jobs: suite: web-frontend - os: debian-12 suite: web-rails + - os: debian-12 + suite: wiki exclude: - suite: apt-repository os: ubuntu-2204 -- 2.39.5 From 5e373778a0e18992a119315538721c7ff52d40a9 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 21 Mar 2025 17:28:52 +0000 Subject: [PATCH 16/16] Use system gem to install dbus-systemd for prometheus --- cookbooks/networking/recipes/default.rb | 4 +++- cookbooks/ruby/attributes/default.rb | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/cookbooks/networking/recipes/default.rb b/cookbooks/networking/recipes/default.rb index 831878ab4..4b1d6d056 100644 --- a/cookbooks/networking/recipes/default.rb +++ b/cookbooks/networking/recipes/default.rb @@ -318,7 +318,9 @@ link "/etc/resolv.conf" do to "../run/systemd/resolve/stub-resolv.conf" end -gem_package "dbus-systemd" +gem_package "dbus-systemd" do + gem_binary node[:ruby][:system_gem] +end prometheus_exporter "resolved" do port 10028 diff --git a/cookbooks/ruby/attributes/default.rb b/cookbooks/ruby/attributes/default.rb index 194f29957..39ac754e5 100644 --- a/cookbooks/ruby/attributes/default.rb +++ b/cookbooks/ruby/attributes/default.rb @@ -1,5 +1,16 @@ default[:ruby][:fullstaq] = true +default[:ruby][:system_version] = if platform?("debian") + "3.1" + elsif node[:lsb][:release].to_f < 22.04 + "2.7" + else + "3.0" + end +default[:ruby][:system_interpreter] = "/usr/bin/ruby#{node[:ruby][:system_version]}" +default[:ruby][:system_gem] = "/usr/bin/gem#{node[:ruby][:system_version]}" +default[:ruby][:system_bundle] = "/usr/bin/bundle#{node[:ruby][:system_version]}" + if node[:ruby][:fullstaq] default[:ruby][:version] = "3.4" @@ -9,15 +20,9 @@ if node[:ruby][:fullstaq] else - default[:ruby][:version] = if platform?("debian") - "3.1" - elsif node[:lsb][:release].to_f < 22.04 - "2.7" - else - "3.0" - end - default[:ruby][:interpreter] = "/usr/bin/ruby#{node[:ruby][:version]}" - default[:ruby][:gem] = "/usr/bin/gem#{node[:ruby][:version]}" - default[:ruby][:bundle] = "/usr/bin/bundle#{node[:ruby][:version]}" + default[:ruby][:version] = node[:ruby][:system_version] + default[:ruby][:interpreter] = node[:ruby][:system_interpreter] + default[:ruby][:gem] = node[:ruby][:system_gem] + default[:ruby][:bundle] = node[:ruby][:system_bundle] end -- 2.39.5