From: Tom Hughes Date: Mon, 10 Mar 2025 00:44:45 +0000 (+0000) Subject: Merge remote-tracking branch 'github/pull/749' X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/HEAD?hp=66e478e54a6fa9737402413c8eb919ae8ec64c7a Merge remote-tracking branch 'github/pull/749' --- diff --git a/.kitchen.yml b/.kitchen.yml index 3e2206f7f..6f4eb6bb6 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -1,7 +1,8 @@ --- driver: name: dokken - chef_version: 18 + chef_image: ghcr.io/firefishy/chef-docker-image + chef_version: latest volumes: - /var/lib/docker env: @@ -21,8 +22,6 @@ provisioner: name: dokken chef_license: accept data_bags_path: test/data_bags - chef_image: ghcr.io/firefishy/chef-docker-image:latest - chef_version: latest slow_resource_report: true clean_dokken_sandbox: true attributes: @@ -94,6 +93,9 @@ suites: - name: blogs run_list: - recipe[blogs::default] + attributes: + ruby: + version: 3.3 - name: chef run_list: - recipe[chef::default] @@ -403,6 +405,8 @@ suites: run_list: - recipe[taginfo::default] attributes: + ruby: + version: 3.3 taginfo: sites: - name: taginfo.example.com diff --git a/README.md b/README.md index 86145db17..6b6304834 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This repository manages the configuration of all the servers run by the OpenStreetMap Foundation's Operations Working Group. We use -[Chef](https://www.chef.io/) to automated the configuration of all of our +[Chef](https://www.chef.io/) to automate the configuration of all of our servers. [OSMF Operations Working Group](https://operations.osmfoundation.org/) @@ -14,20 +14,20 @@ servers. We make extensive use of roles to configure the servers. In general we have: -## Server-specific roles (e.g. [faffy.rb](roles/faffy.rb)) +## Server-specific roles (e.g., [faffy.rb](roles/faffy.rb)) These deal with particular setup or quirks of a server, such as its IP address. They also include roles representing the service they are performing, and the location they are in and any particular hardware they have that needs configuration. All our servers are [named after dragons](https://wiki.openstreetmap.org/wiki/Servers/Name_Ideas). -## Hardware-specific roles (e.g. [hp-g9.rb](roles/hp-g9.rb)) +## Hardware-specific roles (e.g., [hp-g9.rb](roles/hp-g9.rb)) Covers anything specific to a certain piece of hardware, like a motherboard, that could apply to multiple machines. -## Location-specific roles (e.g. [equinix-dub.rb](roles/equinix-dub.rb)) +## Location-specific roles (e.g., [equinix-dub.rb](roles/equinix-dub.rb)) These form a hierarchy of datacentres, organisations, and countries where our servers are located. -## Service-specific roles (e.g. [web-frontend](roles/web-frontend.rb)) +## Service-specific roles (e.g., [web-frontend](roles/web-frontend.rb)) These cover the services that the server is running, and will include the recipes required for that service along with any specific configurations and other cascading roles. diff --git a/cookbooks/apt/recipes/fullstaq-ruby.rb b/cookbooks/apt/recipes/fullstaq-ruby.rb new file mode 100644 index 000000000..d9dad7d37 --- /dev/null +++ b/cookbooks/apt/recipes/fullstaq-ruby.rb @@ -0,0 +1,27 @@ +# +# Cookbook:: apt +# Recipe:: fullstaq-ruby +# +# Copyright:: 2025, Tom Hughes +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "apt" + +apt_repository "fullstaq-ruby" do + uri "https://apt.fullstaqruby.org" + distribution "#{node[:platform]}-#{node[:platform_version]}" + components ["main"] + key "https://raw.githubusercontent.com/fullstaq-ruby/server-edition/main/fullstaq-ruby.asc" +end diff --git a/cookbooks/bind/templates/default/db.10.erb b/cookbooks/bind/templates/default/db.10.erb index 06aef6442..298811c68 100644 --- a/cookbooks/bind/templates/default/db.10.erb +++ b/cookbooks/bind/templates/default/db.10.erb @@ -2,7 +2,7 @@ $TTL 604800 @ IN SOA <%= node[:fdqn] %>. root.openstreetmap.org. ( - 2021092001 ; Serial + 2025021801 ; Serial 604800 ; Refresh 86400 ; Retry 2419200 ; Expire @@ -44,6 +44,7 @@ $TTL 604800 100.48.0 IN PTR pdu1.ams.openstreetmap.org. 101.48.0 IN PTR pdu2.ams.openstreetmap.org. 102.48.0 IN PTR oob1.ams.openstreetmap.org. +103.48.0 IN PTR ats1.ams.openstreetmap.org. 3.49.0 IN PTR faffy.oob.openstreetmap.org. 4.49.0 IN PTR dribble.oob.openstreetmap.org. diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index 0fb2cc7aa..5c650f242 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -44,10 +44,18 @@ git "/srv/blogs.openstreetmap.org" do group "blogs" end +bundle_config "/srv/blogs.openstreetmap.org" do + action :nothing + user "blogs" + group "blogs" + settings "deployment" => "true", + "without" => "development:test", + "build.sqlite3" => "--enable-system-libraries" + subscribes :create, "git[/srv/blogs.openstreetmap.org]", :immediately +end + bundle_install "/srv/blogs.openstreetmap.org" do action :nothing - options "--deployment --without development test" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately @@ -56,7 +64,6 @@ end bundle_exec "/srv/blogs.openstreetmap.org" do action :nothing command "pluto build -t osm -o build" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately diff --git a/cookbooks/blogs/templates/default/blogs-update.erb b/cookbooks/blogs/templates/default/blogs-update.erb index a7d021495..15cc82483 100644 --- a/cookbooks/blogs/templates/default/blogs-update.erb +++ b/cookbooks/blogs/templates/default/blogs-update.erb @@ -2,8 +2,6 @@ cd /srv/blogs.openstreetmap.org -export BUNDLE_PATH="vendor/bundle" - <%= node[:ruby][:bundle] %> exec pluto \ --quieter \ --config=/srv/blogs.openstreetmap.org build \ diff --git a/cookbooks/community/recipes/default.rb b/cookbooks/community/recipes/default.rb index 000a57a7e..0628d883a 100644 --- a/cookbooks/community/recipes/default.rb +++ b/cookbooks/community/recipes/default.rb @@ -62,9 +62,8 @@ end git "/srv/community.openstreetmap.org/docker" do action :sync repository "https://github.com/discourse/discourse_docker.git" - # Revision pin not possible as launch wrapper automatically updates git repo. - revision "main" - depth 1 + # DANGER launch wrapper automatically updates git repo if rebuild method used: https://github.com/discourse/discourse_docker/blob/107ffb40fe8b1ea40e00814468db974a4f3f8e8f/launcher#L799 + revision "3715498fc188d60c0b579443383c4e973cf26f59" user "root" group "root" notifies :run, "notify_group[discourse_container_new_data]" @@ -122,18 +121,24 @@ notify_group "discourse_container_new_web_only" do notifies :run, "execute[discourse_container_data_start]", :immediately # noop if site up notifies :run, "execute[discourse_container_web_only_bootstrap]", :immediately # site up but runs in parallel. Slow notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down - notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down + notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down + notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down + notifies :run, "execute[discourse_container_data_start]", :immediately # site down notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore end notify_group "discourse_container_new_data" do notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down - notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down + notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down + notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down + notifies :run, "execute[discourse_container_data_start]", :immediately # site down notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore end notify_group "discourse_container_new_mail_receiver" do - notifies :run, "execute[discourse_container_mail_receiver_rebuild]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_destroy]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_bootstrap]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_start]", :immediately end # Attempt at a failsafe to ensure all containers are running @@ -144,17 +149,26 @@ notify_group "discourse_container_ensure_all_running" do notifies :run, "execute[discourse_container_mail_receiver_start]", :delayed end -execute "discourse_container_data_start" do +execute "discourse_container_data_bootstrap" do action :nothing - command "./launcher start data" + command "./launcher bootstrap data" + cwd "/srv/community.openstreetmap.org/docker/" + user "root" + group "root" + retries 2 # Postgres upgrades required a second run +end + +execute "discourse_container_data_destroy" do + action :nothing + command "./launcher destroy data" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" end -execute "discourse_container_data_rebuild" do +execute "discourse_container_data_start" do action :nothing - command "./launcher rebuild data" + command "./launcher start data" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" @@ -184,10 +198,17 @@ execute "discourse_container_web_only_start" do group "root" end -# Rebuild: Stop Destroy Bootstap Start -execute "discourse_container_mail_receiver_rebuild" do +execute "discourse_container_mail_receiver_bootstrap" do + action :nothing + command "./launcher bootstrap mail-receiver" + cwd "/srv/community.openstreetmap.org/docker/" + user "root" + group "root" +end + +execute "discourse_container_mail_receiver_destroy" do action :nothing - command "./launcher rebuild mail-receiver" + command "./launcher destroy mail-receiver" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" diff --git a/cookbooks/community/templates/default/data.yml.erb b/cookbooks/community/templates/default/data.yml.erb index c5c59d288..37573874e 100644 --- a/cookbooks/community/templates/default/data.yml.erb +++ b/cookbooks/community/templates/default/data.yml.erb @@ -3,7 +3,7 @@ # templates: - - "templates/postgres.13.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW + - "templates/postgres.15.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW - "templates/redis.template.yml" # any extra arguments for Docker? @@ -51,6 +51,6 @@ hooks: run: # Make sure this matches the postgresql version template above - replace: - filename: "/etc/postgresql/13/main/postgresql.conf" + filename: "/etc/postgresql/15/main/postgresql.conf" from: /#?max_connections *=.*/ to: "max_connections = $db_max_connections" diff --git a/cookbooks/community/templates/default/web_only.yml.erb b/cookbooks/community/templates/default/web_only.yml.erb index 5beef4802..ff9127ff1 100644 --- a/cookbooks/community/templates/default/web_only.yml.erb +++ b/cookbooks/community/templates/default/web_only.yml.erb @@ -19,7 +19,7 @@ links: # any extra arguments for Docker? # docker_args: -# Latest Version v3.3.3 +# Latest Version v3.4.1 # Discourse only support tests-passed and stable branches params: version: stable @@ -116,14 +116,6 @@ hooks: cmd: - sudo -H -E -u discourse cp /shared/feeds/update-feeds.atom public/update-feeds.atom after_ssl: - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /listen 80;/ - to: | - listen 80; - listen [::]:80; - rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; - - replace: filename: "/etc/nginx/conf.d/discourse.conf" from: /add_header.+/ @@ -133,3 +125,12 @@ hooks: resolver <%= @resolvers.join(" ") %>; resolver_timeout 5s; ssl_dhparam /shared/ssl/dhparam.pem; + +run: + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /listen 80;/ + to: | + listen 80; + listen [::]:80; + rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; diff --git a/cookbooks/dev/README.md b/cookbooks/dev/README.md index 0253da49b..3918d375b 100644 --- a/cookbooks/dev/README.md +++ b/cookbooks/dev/README.md @@ -2,4 +2,4 @@ This cookbook configures development servers, such as dev.openstreetmap.org. It installs packages required by the users and configures apache for the various -user and api developement sites. +user and api development sites. diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index 073bb3e13..f42c22ee3 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -61,6 +61,7 @@ package %w[ gnuplot-nox golang graphviz + htop irssi jq libargon2-dev @@ -95,6 +96,7 @@ package %w[ lzip lzop mailutils + moreutils make nano ncftp @@ -103,7 +105,6 @@ package %w[ osmium-tool osmosis pandoc - pandoc pbzip2 php-apcu php-cgi @@ -149,6 +150,7 @@ package %w[ unrar unzip whois + xxd zip zlib1g-dev ] @@ -201,7 +203,7 @@ template "/srv/dev.openstreetmap.org/index.html" do end ssl_certificate "dev.openstreetmap.org" do - domains "dev.openstreetmap.org" + domains ["dev.openstreetmap.org", "dev.osm.org"] notifies :reload, "service[apache2]" end @@ -223,7 +225,7 @@ file "/etc/apache2/conf.d/phppgadmin" do end ssl_certificate "phppgadmin.dev.openstreetmap.org" do - domains "phppgadmin.dev.openstreetmap.org" + domains ["phppgadmin.dev.openstreetmap.org", "phppgadmin.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -349,7 +351,7 @@ if node[:postgresql][:clusters][:"15/main"] gpx_directory = "#{site_directory}/gpx" if details[:repository] - site_aliases = details[:aliases] || [] + site_aliases = details[:aliases] || ["#{name}.apis.dev.osm.org"] secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do @@ -570,7 +572,7 @@ if node[:postgresql][:clusters][:"15/main"] end ssl_certificate "apis.dev.openstreetmap.org" do - domains "apis.dev.openstreetmap.org" + domains ["apis.dev.openstreetmap.org", "apis.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -599,7 +601,8 @@ ssl_certificate "ooc.openstreetmap.org" do domains ["ooc.openstreetmap.org", "a.ooc.openstreetmap.org", "b.ooc.openstreetmap.org", - "c.ooc.openstreetmap.org"] + "c.ooc.openstreetmap.org", + "ooc.osm.org"] notifies :reload, "service[apache2]" end diff --git a/cookbooks/dev/templates/default/apache.apis.erb b/cookbooks/dev/templates/default/apache.apis.erb index 4d78fdd6a..470a1fa5f 100644 --- a/cookbooks/dev/templates/default/apache.apis.erb +++ b/cookbooks/dev/templates/default/apache.apis.erb @@ -2,6 +2,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -16,6 +18,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/apis.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.dev.erb b/cookbooks/dev/templates/default/apache.dev.erb index 2d3d9473d..2dd13c2b9 100644 --- a/cookbooks/dev/templates/default/apache.dev.erb +++ b/cookbooks/dev/templates/default/apache.dev.erb @@ -2,6 +2,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -21,6 +23,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.ooc.erb b/cookbooks/dev/templates/default/apache.ooc.erb index 0a9b0b979..a532300e1 100644 --- a/cookbooks/dev/templates/default/apache.ooc.erb +++ b/cookbooks/dev/templates/default/apache.ooc.erb @@ -5,6 +5,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -26,6 +28,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/ooc.openstreetmap.org-access.log combined_extended @@ -37,6 +41,8 @@ ServerName npe.openstreetmap.org + ServerAlias npe.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/npe.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.phppgadmin.erb b/cookbooks/dev/templates/default/apache.phppgadmin.erb index 546a05cbe..67c37df16 100644 --- a/cookbooks/dev/templates/default/apache.phppgadmin.erb +++ b/cookbooks/dev/templates/default/apache.phppgadmin.erb @@ -2,6 +2,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -23,6 +25,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/phppgadmin.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.rails.erb b/cookbooks/dev/templates/default/apache.rails.erb index 957866c5c..082931dde 100644 --- a/cookbooks/dev/templates/default/apache.rails.erb +++ b/cookbooks/dev/templates/default/apache.rails.erb @@ -48,6 +48,7 @@ RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] <% end -%> diff --git a/cookbooks/dev/templates/default/apache.user.erb b/cookbooks/dev/templates/default/apache.user.erb index 373d12258..0c51e44e0 100644 --- a/cookbooks/dev/templates/default/apache.user.erb +++ b/cookbooks/dev/templates/default/apache.user.erb @@ -4,9 +4,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + SSLEngine on SSLCertificateFile /etc/ssl/certs/<%= @user %>.dev.openstreetmap.org.pem SSLCertificateKeyFile /etc/ssl/private/<%= @user %>.dev.openstreetmap.org.key @@ -36,9 +37,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + CustomLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-access.log combined_extended ErrorLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-error.log diff --git a/cookbooks/dhcpd/README.md b/cookbooks/dhcpd/README.md index 3d2a00c7c..f95dbb0d0 100644 --- a/cookbooks/dhcpd/README.md +++ b/cookbooks/dhcpd/README.md @@ -1,3 +1,3 @@ # dhcpd Cookbook -Configures the dhcpd service, which used for the internal network at UCL. +Configures the dhcpd service, which is used for our internal networks. diff --git a/cookbooks/dhcpd/recipes/default.rb b/cookbooks/dhcpd/recipes/default.rb index 0e6b9ec3f..685d08c3b 100644 --- a/cookbooks/dhcpd/recipes/default.rb +++ b/cookbooks/dhcpd/recipes/default.rb @@ -53,7 +53,7 @@ remote_file "/srv/tftp/netboot.xyz.kpxe" do mode "644" end -domain = "#{node[:networking][:roles][:external][:zone]}.openstreetmap.org" +domain = node[:networking][:search].first template "/etc/dhcp/dhcpd.conf" do source "dhcpd.conf.erb" diff --git a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb index 1581475e5..ed4fe2d03 100644 --- a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb +++ b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb @@ -55,6 +55,12 @@ host pdu2.ams.openstreetmap.org { fixed-address 10.0.48.101; } +host ats1.ams.openstreetmap.org { + hardware ethernet 00:c0:b7:e5:5e:f1; + server-name "ats1.ams.openstreetmap.org"; + fixed-address 10.0.48.103; +} + host oob1.dub.openstreetmap.org { hardware ethernet 62:bd:62:a6:05:25; server-name "oob1.dub.openstreetmap.org"; diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index d1b3d159e..0ec59c8b3 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -42,7 +42,7 @@ package %w[ cache_dir = Chef::Config[:file_cache_path] -dnscontrol_version = "4.15.1" +dnscontrol_version = "4.17.0" dnscontrol_arch = if arm? "arm64" diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb index ffc8be609..3558af884 100644 --- a/cookbooks/exim/templates/default/exim4.conf.erb +++ b/cookbooks/exim/templates/default/exim4.conf.erb @@ -753,6 +753,9 @@ signed_smtp: multi_domain = false hosts_try_dane = tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION +<% if node[:exim][:external_interface] -%> + interface = <%= node[:exim][:external_interface] %> +<% end -%> # This transport is used for handling pipe deliveries generated by alias or diff --git a/cookbooks/imagery/recipes/au_act_aerial.rb b/cookbooks/imagery/recipes/au_act_aerial.rb new file mode 100644 index 000000000..cc0135998 --- /dev/null +++ b/cookbooks/imagery/recipes/au_act_aerial.rb @@ -0,0 +1,65 @@ +# +# Cookbook:: imagery +# Recipe:: au_act_aerial +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "act-imagery.openstreetmap.org" do + title "OpenStreetMap - ACT Imagery" + aliases ["act-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-35.942, 148.729], [-35.117, 149.430]] +end + +imagery_layer "act_aerial_imagery_202409" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202409" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + default_layer true + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202311" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202311" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202305" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202305" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end diff --git a/cookbooks/imagery/recipes/au_agri.rb b/cookbooks/imagery/recipes/au_agri.rb index 931ecf3f1..fb011c111 100644 --- a/cookbooks/imagery/recipes/au_agri.rb +++ b/cookbooks/imagery/recipes/au_agri.rb @@ -33,7 +33,7 @@ imagery_layer "au_ga_agri" do copyright "Commonwealth of Australia (Geoscience Australia) - Creative Commons Attribution 4.0 International Licence" background_colour "0 0 0" # Black projection "EPSG:3857" - source "/store/imagery/au/agri/combine.vrt" + source "/store/imagery/au/agri/combine-cutline-cog.tif" max_zoom 17 - revision 1 + revision 3 end diff --git a/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb new file mode 100644 index 000000000..cda5e80d4 --- /dev/null +++ b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb @@ -0,0 +1,59 @@ +# +# Cookbook:: imagery +# Recipe:: au_vic_melbourne_aerial +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "au-vic-melbourne-imagery.openstreetmap.org" do + title "OpenStreetMap - City of Melbourne - Aerial Imagery" + aliases ["au-vic-melbourne-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-37.850667, 144.896981], [-37.775451, 144.991351]] +end + +imagery_layer "melbourne-2020" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2020" + source "/store/imagery/au/city-of-melbourne/CoM_May2020_2cm.cog.tiff" + copyright "(c) 2020 City of Melbourne" + max_zoom 23 + extension "jpeg" + revision 2 + default_layer true +end + +imagery_layer "melbourne-2019" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2019" + source "/store/imagery/au/city-of-melbourne/CoM_03Feb2019.cog.tiff" + copyright "(c) 2019 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end + +imagery_layer "melbourne-2018" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2018" + source "/store/imagery/au/city-of-melbourne/CoM_May2018_10cm.COG.tiff" + copyright "(c) 2018 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end diff --git a/cookbooks/imagery/recipes/tiler.rb b/cookbooks/imagery/recipes/tiler.rb index ec7daa835..571425bda 100644 --- a/cookbooks/imagery/recipes/tiler.rb +++ b/cookbooks/imagery/recipes/tiler.rb @@ -39,9 +39,7 @@ podman_service "titiler" do image container_image volume :"/store/imagery" => "/store/imagery", :"/srv/imagery/sockets" => "/sockets" - environment :BIND => "unix:/sockets/titiler.sock", - :WORKERS_PER_CORE => 1, - :GDAL_CACHEMAX => 200, + environment :GDAL_CACHEMAX => 200, :GDAL_BAND_BLOCK_CACHE => "HASHSET", :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", :GDAL_INGESTED_BYTES_AT_OPEN => 32768, @@ -52,6 +50,7 @@ podman_service "titiler" do :VSI_CACHE_SIZE => 5000000, :TITILER_API_ROOT_PATH => "/api/v1/titiler", :FORWARDED_ALLOW_IPS => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips + command "gunicorn -k uvicorn.workers.UvicornWorker titiler.application.main:app --bind unix:/sockets/titiler.sock --workers #{node.cpu_cores}" end systemd_service "titiler-restart" do @@ -64,7 +63,7 @@ end systemd_timer "titiler-restart" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "2h" randomized_delay_sec "20m" end diff --git a/cookbooks/imagery/recipes/za_ngi_topo.rb b/cookbooks/imagery/recipes/za_ngi_topo.rb index a1fd92c61..ac5a28b37 100644 --- a/cookbooks/imagery/recipes/za_ngi_topo.rb +++ b/cookbooks/imagery/recipes/za_ngi_topo.rb @@ -32,6 +32,7 @@ imagery_layer "za_ngi_topo_250k" do source "/store/imagery/za/ngi-topo-250k/ngi-topo-250k-combined.vrt" copyright 'State Copyright © 1996–2010 Chief Directorate: National Geo-spatial Information' default_layer true + revision 2 end imagery_layer "za_ngi_topo_50k" do @@ -40,4 +41,5 @@ imagery_layer "za_ngi_topo_50k" do projection "EPSG:3857" source "/store/imagery/za/ngi-topo-50k/ngi-topo-50k-combined.vrt" copyright 'State Copyright © 1996–2013 Chief Directorate: National Geo-spatial Information' + revision 2 end diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index 5ce7d2b88..7151c68c6 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -105,14 +105,19 @@ action :create do description "Map server for #{new_resource.site} layer" environment "MS_DEBUGLEVEL" => "0", "MS_ERRORFILE" => "stderr", - "GDAL_CACHEMAX" => "128" + "GDAL_CACHEMAX" => "128", + "GDAL_HTTP_TCP_KEEPALIVE" => "YES", + "GDAL_HTTP_VERSION" => "2TLS", + "GDAL_ENABLE_WMS_CACHE" => "NO" limit_nofile 16384 - memory_max "4G" + memory_high "12G" + memory_max "12G" + limit_core 0 user "imagery" group "imagery" exec_start "/usr/bin/multiwatch -f 8 --signal=TERM -- /usr/lib/cgi-bin/mapserv" standard_input "socket" - sandbox true + sandbox :enable_network => true restrict_address_families "AF_UNIX" timeout_stop_sec 60 not_if { new_resource.uses_tiler } @@ -153,7 +158,7 @@ action :create do systemd_timer "mapserv-fcgi-#{new_resource.site}-stop" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "6h" randomized_delay_sec "20m" not_if { new_resource.uses_tiler } end diff --git a/cookbooks/imagery/templates/default/mapserver.map.erb b/cookbooks/imagery/templates/default/mapserver.map.erb index c918ca9f1..728723fcc 100644 --- a/cookbooks/imagery/templates/default/mapserver.map.erb +++ b/cookbooks/imagery/templates/default/mapserver.map.erb @@ -64,6 +64,5 @@ MAP STATUS DEFAULT TYPE RASTER PROCESSING "RESAMPLE=AVERAGE" - PROCESSING "CLOSE_CONNECTION=DEFER" END # layer END diff --git a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb index 03a439343..a232d7830 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb @@ -23,8 +23,11 @@ upstream <%= @name %>_fastcgi { <% end -%> server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; + server_name <%= @name %> a.<%= @name %> b.<%= @name %> c.<%= @name %><% @aliases.each do |alias_name| %> <%= alias_name %> a.<%= alias_name %> b.<%= alias_name %> c.<%= alias_name %><%- end -%>; http2_max_concurrent_streams 512; diff --git a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb index c85dd87bb..adc2d23c0 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb @@ -1,11 +1,9 @@ <% require 'uri' %> # DO NOT EDIT - This file is being maintained by Chef -location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { +location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" { <% if @uses_tiler -%> set $args ""; - rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.jpg /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=jpeg break; - rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.jpeg /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=jpeg break; - rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.png /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=png break; + rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp) /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x.$4?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first break; proxy_pass http://<%= @site %>_tiler_backend; proxy_set_header Host $host; proxy_set_header Referer $http_referer; @@ -17,7 +15,7 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { proxy_set_header Cache-Control ""; proxy_set_header Pragma ""; proxy_redirect off; - proxy_cache_key "<%= @layer %><%= @revision %> $request_method $1 $2 $3"; + proxy_cache_key "<%= @layer %><%= @revision %> $request_method $1 $2 $3 $4"; proxy_cache proxy_cache_zone; proxy_cache_valid 200 204 180d; proxy_cache_use_stale error timeout updating http_502 http_503 http_504; @@ -68,8 +66,8 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { gzip off; # Add HTTP Cache-Control + Expires Headers - expires 7d; - add_header Cache-Control "stale-while-revalidate=604800, stale-if-error=604800"; + expires 31d; + add_header Cache-Control "stale-while-revalidate=2678400, stale-if-error=2678400"; add_header x-cache-status $upstream_cache_status; # Allow CORS requests @@ -77,8 +75,8 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { } <% if @root_layer -%> -rewrite "^/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; +rewrite "^/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; <% end -%> <% @url_aliases.each do |url| -%> -rewrite "^<%= url %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; +rewrite "^<%= url %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; <% end -%> diff --git a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb index 155620949..055c2f849 100644 --- a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb @@ -21,8 +21,11 @@ upstream titiler_api_backend { } server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; + server_name <%= @name %> <% @aliases.each do |alias_name| %> <%= alias_name %><%- end -%>; http2_max_concurrent_streams 512; diff --git a/cookbooks/letsencrypt/files/default/bin/renew-hook b/cookbooks/letsencrypt/files/default/bin/deploy-hook similarity index 100% rename from cookbooks/letsencrypt/files/default/bin/renew-hook rename to cookbooks/letsencrypt/files/default/bin/deploy-hook diff --git a/cookbooks/letsencrypt/files/default/bin/renew b/cookbooks/letsencrypt/files/default/bin/renew index 6a0482185..f3c7cd0cd 100755 --- a/cookbooks/letsencrypt/files/default/bin/renew +++ b/cookbooks/letsencrypt/files/default/bin/renew @@ -1,10 +1,9 @@ #!/bin/sh -cd /srv/acme.openstreetmap.org - /usr/bin/certbot renew \ + --key-type ecdsa \ --quiet \ --config-dir /srv/acme.openstreetmap.org/config \ --work-dir /srv/acme.openstreetmap.org/work \ --logs-dir /srv/acme.openstreetmap.org/logs \ - --renew-hook /srv/acme.openstreetmap.org/bin/renew-hook + --deploy-hook /srv/acme.openstreetmap.org/bin/deploy-hook diff --git a/cookbooks/letsencrypt/metadata.rb b/cookbooks/letsencrypt/metadata.rb index 52ef99f7c..a826ce5e4 100644 --- a/cookbooks/letsencrypt/metadata.rb +++ b/cookbooks/letsencrypt/metadata.rb @@ -9,3 +9,4 @@ supports "ubuntu" depends "accounts" depends "apache" depends "chef" +depends "ruby" diff --git a/cookbooks/letsencrypt/recipes/default.rb b/cookbooks/letsencrypt/recipes/default.rb index 382a0a58c..3e492f6af 100644 --- a/cookbooks/letsencrypt/recipes/default.rb +++ b/cookbooks/letsencrypt/recipes/default.rb @@ -20,13 +20,11 @@ include_recipe "accounts" include_recipe "apache" include_recipe "chef::knife" +include_recipe "ruby" keys = data_bag_item("chef", "keys") -package %w[ - certbot - ruby -] +package "certbot" directory "/etc/letsencrypt" do owner "letsencrypt" @@ -116,6 +114,13 @@ remote_directory "/srv/acme.openstreetmap.org/bin" do files_mode "755" end +template "/srv/acme.openstreetmap.org/bin/upload" do + source "upload.erb" + owner "root" + group "root" + mode "755" +end + directory "/srv/acme.openstreetmap.org/requests" do owner "root" group "root" @@ -168,6 +173,13 @@ Dir.glob("*", :base => "/srv/acme.openstreetmap.org/requests") do |name| end end +template "/srv/acme.openstreetmap.org/bin/check-certificate" do + source "check-certificate.erb" + owner "root" + group "root" + mode "755" +end + template "/srv/acme.openstreetmap.org/bin/check-certificates" do source "check-certificates.erb" owner "root" diff --git a/cookbooks/letsencrypt/files/default/bin/check-certificate b/cookbooks/letsencrypt/templates/default/check-certificate.erb old mode 100755 new mode 100644 similarity index 93% rename from cookbooks/letsencrypt/files/default/bin/check-certificate rename to cookbooks/letsencrypt/templates/default/check-certificate.erb index f24681589..319072b6b --- a/cookbooks/letsencrypt/files/default/bin/check-certificate +++ b/cookbooks/letsencrypt/templates/default/check-certificate.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "socket" require "openssl" @@ -33,6 +33,10 @@ if ssl puts "Certificate #{domains.first} on #{host} expires at #{certificate.not_after}" end + unless certificate.public_key.is_a?(OpenSSL::PKey::EC) + puts "Certificate #{domains.first} on #{host} does not use ECDSA key type" + end + digest = OpenSSL::Digest::SHA1.new certificate_id = OpenSSL::OCSP::CertificateId.new(certificate, issuer, digest) ocsp_request = OpenSSL::OCSP::Request.new.add_certid(certificate_id) diff --git a/cookbooks/letsencrypt/templates/default/request.erb b/cookbooks/letsencrypt/templates/default/request.erb index eaefa5bbe..65e80b303 100644 --- a/cookbooks/letsencrypt/templates/default/request.erb +++ b/cookbooks/letsencrypt/templates/default/request.erb @@ -2,6 +2,11 @@ # DO NOT EDIT - This file is being maintained by Chef +if [ "$(id -un)" != "letsencrypt" ]; then + echo "Error: This script must be run as user letsencrypt" >&2 + exit 1 +fi + /usr/bin/certbot certonly \ --non-interactive \ --config-dir /srv/acme.openstreetmap.org/config \ @@ -10,12 +15,12 @@ --email operations@osmfoundation.org \ --agree-tos \ --expand \ + --renew-with-new-domains \ + --cert-name <%= @domains.first %> \ <% @domains.each do |domain| -%> --domain <%= domain %> \ <% end -%> --webroot \ - --webroot-path /srv/acme.openstreetmap.org/html - -/srv/acme.openstreetmap.org/bin/upload \ - <%= @domains.first %> \ - /srv/acme.openstreetmap.org/config/live/<%= @domains.first %> + --webroot-path /srv/acme.openstreetmap.org/html \ + --deploy-hook /srv/acme.openstreetmap.org/bin/deploy-hook \ + "$@" diff --git a/cookbooks/letsencrypt/files/default/bin/upload b/cookbooks/letsencrypt/templates/default/upload.erb old mode 100755 new mode 100644 similarity index 67% rename from cookbooks/letsencrypt/files/default/bin/upload rename to cookbooks/letsencrypt/templates/default/upload.erb index a52f30257..e89912c4f --- a/cookbooks/letsencrypt/files/default/bin/upload +++ b/cookbooks/letsencrypt/templates/default/upload.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "json" require "tempfile" @@ -17,4 +17,4 @@ file = Tempfile.new(["letsencrypt", ".json"]) file.puts JSON.generate(bag) file.close -system("/opt/chef/embedded/bin/knife", "data", "bag", "from", "file", "letsencrypt", file.path) +system("/opt/chef/embedded/bin/knife", "data", "bag", "from", "file", "letsencrypt", file.path, "--config", "/srv/acme.openstreetmap.org/.chef/knife.rb", "--key", "/srv/acme.openstreetmap.org/.chef/client.pem") diff --git a/cookbooks/mediawiki/recipes/default.rb b/cookbooks/mediawiki/recipes/default.rb index c7249e8fa..d784e7b7f 100644 --- a/cookbooks/mediawiki/recipes/default.rb +++ b/cookbooks/mediawiki/recipes/default.rb @@ -41,6 +41,7 @@ package %w[ composer unzip ffmpeg + firejail ] # Mediawiki enhanced difference engine diff --git a/cookbooks/mediawiki/resources/site.rb b/cookbooks/mediawiki/resources/site.rb index 0bbddfe81..096484a99 100644 --- a/cookbooks/mediawiki/resources/site.rb +++ b/cookbooks/mediawiki/resources/site.rb @@ -425,6 +425,7 @@ action :create do if new_resource.commons mediawiki_extension "QuickInstantCommons" do site new_resource.site + template "mw-ext-QuickInstantCommons.inc.php.erb" update_site false end else diff --git a/cookbooks/mediawiki/templates/default/composer.local.json.erb b/cookbooks/mediawiki/templates/default/composer.local.json.erb index 73678e1cc..338a0de9d 100644 --- a/cookbooks/mediawiki/templates/default/composer.local.json.erb +++ b/cookbooks/mediawiki/templates/default/composer.local.json.erb @@ -8,6 +8,13 @@ } }, "require": { - "guzzlehttp/psr7": "2.4.5" + "guzzlehttp/psr7": "2.4.5", + "data-values/common": "1.0.0", + "data-values/data-values": "3.0.0", + "data-values/geo": "4.2.3", + "data-values/interfaces": "1.0.0", + "data-values/number": "0.11.1", + "data-values/serialization": "1.2.4", + "data-values/time": "1.0.4" } } diff --git a/cookbooks/mediawiki/templates/default/mw-ext-QuickInstantCommons.inc.php.erb b/cookbooks/mediawiki/templates/default/mw-ext-QuickInstantCommons.inc.php.erb new file mode 100644 index 000000000..06151ed58 --- /dev/null +++ b/cookbooks/mediawiki/templates/default/mw-ext-QuickInstantCommons.inc.php.erb @@ -0,0 +1,18 @@ + '\MediaWiki\Extension\QuickInstantCommons\Repo', + 'name' => 'wikimediacommons', + 'directory' => $wgUploadDirectory, + 'apibase' => 'https://commons.wikimedia.org/w/api.php', + 'hashLevels' => 2, + 'thumbUrl' => 'https://upload.wikimedia.org/wikipedia/commons/thumb', + 'fetchDescription' => true, + 'descriptionCacheExpiry' => 60*60*24*30, + 'transformVia404' => true, + 'abbrvThreshold' => 160, + 'apiMetadataExpiry' => 60*60*24*30, + 'disabledMediaHandlers' => [TiffHandler::class] +]; diff --git a/cookbooks/networking/attributes/default.rb b/cookbooks/networking/attributes/default.rb index 9832ce8f3..06511d462 100644 --- a/cookbooks/networking/attributes/default.rb +++ b/cookbooks/networking/attributes/default.rb @@ -6,11 +6,10 @@ default[:networking][:firewall][:outgoing] = [] default[:networking][:firewall][:http_rate_limit] = nil default[:networking][:firewall][:http_connection_limit] = nil default[:networking][:firewall][:allowlist] = [] -default[:networking][:roles] = {} default[:networking][:interfaces] = {} default[:networking][:nameservers] = %w[8.8.8.8 8.8.4.4 2001:4860:4860::8888 2001:4860:4860::8844] default[:networking][:search] = [] -default[:networking][:dnssec] = "allow-downgrade" +default[:networking][:dnssec] = "false" default[:networking][:hostname] = node.name default[:networking][:wireguard][:enabled] = true default[:networking][:wireguard][:keepalive] = 180 diff --git a/cookbooks/networking/recipes/default.rb b/cookbooks/networking/recipes/default.rb index 2b4d96761..831878ab4 100644 --- a/cookbooks/networking/recipes/default.rb +++ b/cookbooks/networking/recipes/default.rb @@ -59,37 +59,18 @@ interfaces = node[:networking][:interfaces].collect do |name, interface| [interface[:interface], name] end.to_h -node[:networking][:interfaces].each do |name, interface| - if interface[:interface] =~ /^(.*)\.(\d+)$/ - vlan_interface = Regexp.last_match(1) - vlan_id = Regexp.last_match(2) - - parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" +node[:networking][:interfaces].each do |_, interface| + next unless interface[:interface] =~ /^(.*)\.(\d+)$/ - node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface - node.default_unless[:networking][:interfaces][parent][:vlans] = [] + vlan_interface = Regexp.last_match(1) + vlan_id = Regexp.last_match(2) - node.default[:networking][:interfaces][parent][:vlans] << vlan_id - end + parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" - next unless interface[:role] && (role = node[:networking][:roles][interface[:role]]) - - if interface[:inet] && role[:inet] - node.default_unless[:networking][:interfaces][name][:inet][:prefix] = role[:inet][:prefix] - node.default_unless[:networking][:interfaces][name][:inet][:gateway] = role[:inet][:gateway] - node.default_unless[:networking][:interfaces][name][:inet][:routes] = role[:inet][:routes] - node.default_unless[:networking][:interfaces][name][:inet][:rules] = role[:inet][:rules] - end + node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface + node.default_unless[:networking][:interfaces][parent][:vlans] = [] - if interface[:inet6] && role[:inet6] - node.default_unless[:networking][:interfaces][name][:inet6][:prefix] = role[:inet6][:prefix] - node.default_unless[:networking][:interfaces][name][:inet6][:gateway] = role[:inet6][:gateway] - node.default_unless[:networking][:interfaces][name][:inet6][:routes] = role[:inet6][:routes] - node.default_unless[:networking][:interfaces][name][:inet6][:rules] = role[:inet6][:rules] - end - - node.default_unless[:networking][:interfaces][name][:metric] = role[:metric] - node.default_unless[:networking][:interfaces][name][:zone] = role[:zone] + node.default[:networking][:interfaces][parent][:vlans] << vlan_id end node[:networking][:interfaces].each do |_, interface| @@ -103,6 +84,8 @@ node[:networking][:interfaces].each do |_, interface| notifies :run, "notify_group[networkctl-reload]" end elsif interface[:interface] =~ /^bond\d+$/ + next unless interface[:bond] + template "/etc/systemd/network/10-#{interface[:interface]}.netdev" do source "bond.netdev.erb" owner "root" @@ -335,9 +318,7 @@ link "/etc/resolv.conf" do to "../run/systemd/resolve/stub-resolv.conf" end -gem_package "dbus-systemd" do - gem_binary node[:ruby][:gem] -end +gem_package "dbus-systemd" prometheus_exporter "resolved" do port 10028 diff --git a/cookbooks/networking/templates/default/network.erb b/cookbooks/networking/templates/default/network.erb index 7f38de3c5..f837c65a5 100644 --- a/cookbooks/networking/templates/default/network.erb +++ b/cookbooks/networking/templates/default/network.erb @@ -91,6 +91,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> @@ -107,6 +110,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> diff --git a/cookbooks/openssh/recipes/default.rb b/cookbooks/openssh/recipes/default.rb index 8b57aaaef..98e2fa9f2 100644 --- a/cookbooks/openssh/recipes/default.rb +++ b/cookbooks/openssh/recipes/default.rb @@ -43,7 +43,7 @@ hosts = search(:node, "networking:interfaces").sort_by { |n| n[:hostname] }.coll names = [name] unless node.interfaces(:role => :internal).empty? - names.unshift("#{name}.#{node[:networking][:roles][:external][:zone]}.openstreetmap.org") + names.unshift("#{name}.#{node[:networking][:search].first}") end unless node.interfaces(:role => :external).empty? diff --git a/cookbooks/openssh/templates/default/ssh_known_hosts.erb b/cookbooks/openssh/templates/default/ssh_known_hosts.erb index c2d996a75..e0f173c06 100644 --- a/cookbooks/openssh/templates/default/ssh_known_hosts.erb +++ b/cookbooks/openssh/templates/default/ssh_known_hosts.erb @@ -20,11 +20,12 @@ yevaud.oob.openstreetmap.org,yevaud.oob,10.0.1.15 ssh-rsa AAAAB3NzaC1yc2EAAAABIw switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCbUecW/591rGCYtkdVTUgAKTBc+lEhraeMCau7CHqMkkUn+/pe68toDySdTaO8jua1q1asuEWH3MZ2AGAPSF+xhrqJLLPQmm0WdEAQfZChxQUoqCPqlWAuM5u+bAc3ntKjw2v2myoGGAqjL27QkZDP8qPdxvKcbm2YpPVOHP56jsyp/9BjVhIpWVmMhkOAfBSjdKU+uP+NueHrJ/AueoqucQE0txeo5Mmw7GlRuLTsMNDU3Khlhjs/2Q0QGiMXV6hQxj5AgnaDZMdSRbp8lXlctDZmipr7jVk3TL+knzySih5wJ7wmeZimW8Dhcxk6HBo2lGgPGdtQgttCF5ZUGcfF switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFPNVsMuGWMe2OP9AIg0hqZLSJP8zyNHWqYrXd5Mnfr12XdhzWEAKLTD14ZnJcz/GBaE0ad3IjT+hVoS/GVtxpQ= switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA615MNtLzOv6qPfFsD0I6Pd0itshpHLAVImjt1ME4SM -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2oeoe4olettjDauLuRiXRQVg5k6EO+PFfS9Ho3OzlOeYr1+h1SSPLkFtr20yOKcGXQcmYLjTbzvt/sbz2JUbfJHc2n7tMvQKK5ClVxLwqt2jpfSepGbG2KjG6vwxpkcDMH0bquzr/8ehpYmBwh3m8EARgcVNYoVL6J5OV4ILAtjNEllsrB71gitrarC8iqslRLVMjane9YKa7z0AGkwcT9w4qbcyuQ24rJF0Xo/IcraxVAYeX0Xtfx9rD3Zt1kHawaKYQdJM7RZvTtWcgCzHt738lFZ15Qj3at6vxxGpnduZiuHLB3PCW9yP4CFHm3Qs3TUyh3VqrNVFFdFvUXnP+DR3QK2E7TpViagZmXb+2AeCaLtrJ+1h0NEmQmliB4Zm4XKPsdS7k6Ul43pYQ5i1ipwZBNvhJrByQJ6UkpIjrwUlQBxXcEWGZaSL0sqqDQIYPizXZWOh7pQay2LxnHUVp0MZT30iVLYjptu9PcJ6CBwoKzhnVHPh7SON4uzDyMrk= -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFLujNadDWF9OqF//izQ3/jm4zElWb2nMjfpeiUVCYh/Jrsl07c1bRJlnSmoxL2H0xeHJXQh1E4PPy4o/5N4GfE= -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINxBm8YPJNtUIDh+0YWKY/5XKv8QKQBtF3ecqQcxIQNJ +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTgHga8l3mOpydchsZsvlaCJpxHoUV5dYBzK8m3iOedqd1UBS9KbHEfgFOvXcd3cmOnQT5dM1YXCQ74WB9hI5Wbkz+oHxWq+ZJeHOsXVIqxFzgxsI8e8gxJve7VyzxR52yiStBdLODhwwMmRPIH8AzzR93Fzi5AAQrGoHoqi/R0A/l6yTI8YnjJ2WaKzflId0YMyjR2KhOj+jCgCZIJR6DqfVpZXERMAtdVUY78Ub0E8iYyWg24gI176l4jCezmaTZ99tWo41mGXax9zXizQr4jx4yIxzRMhqAoJk8oDBkdhxudRUL2eWvBs3v1kcY3Ywtxi+wQIqClvQgBkWrHe4Jujjl8EEFuGHh6156Al7WqEnC9EcENA5Lk/IM0lzDbeT+WTyO/qU8cu035hmPMydYBHjqzo5L6yDwFXEAtJpHlr8QQDzEwgtK9bXc453cQDEJW4JWjP7UlsTlbflSBGQrY5FpI8D+MgnDYNld2kD4Fxzomf4ne+M0w19jrtNMJuE= +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPkUAdFYwolSkUs7d1k+mRVBCV7WRrKsebj8RbjLqPCmEQo9ABSy2iAuYOYgkkqlgmB8F8OHsnNtmGzx7smCwAU= +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKVdlLkjFUDExO7YCixEWvaKHGS0IQHn2nBrkMjGTwRx +ats1.ams.openstreetmap.org,ats1.ams,10.0.48.103 ssh-rsa AAAAB3NzaC1yc2EAAAACAQEAAAEBAKnx+hwj8Am4DuYXZLK1whHqOjW4Bbs/KrLqFokoZgJxzp1g+6sOAzXPy8w3dYyHBJaXCmr3RisIhG7fj0pITkywhU8ygiaEHDzPFyGqxVYe1oI3ayE47hC14Jhu33p7F+pY1xEVUa2H3Xu4F8toHnmuIjLAJ9phaEVvnjLl5n47dM1VfB5KB4NvuDejBthaZpgbALainUuKaqUL2Wgb0LxbDFRTQ+r6uUY4SEFnfqwveovdBL/edd+14/yRob0/r6zlubi5PhvMH1rCaE7flFNr61LItz06VAs+vZpLTNTIFXHqAtCjIA6e8VvvOn8wwmjxXKLsUOFZipvW9Uc9kTk= faffy.oob.openstreetmap.org,faffy.oob,10.0.49.3 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuL6kBDuK+/xj40jp4ZP9HVoH1enU1W9MWO4Q0cxlD9sjHaz/mIbLhtPxElrw807QLsUdOx1IeD0jCLi9XAYYEF2xFFZ9DIP+qGp6hQ3XJrmOVgbDuckPjdvJWAxC5TVaWqwvHyp02biJCfSbOzsmjm2cOp03Cqq18HJOCfA3AxQ/zSQDSVemFoFPVeM67M9NgM64z1idPAvMVtW8Kd+unU/oE7V+Jil5TR5vwK+kVKEixxMtZfkJn0bNKVFWX5sJYmKWze+b95O+iQlsxmQW/p7UVM/3krKdBBT8+9b6UOv8TN0Yhopp86aXEgtvUOOyyimjqO3V5ogRMGj3b9G+9 -vhagar.oob.openstreetmap.org,10.0.49.5 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJaXI71RCPjbni+WxBnOf2CzkLi0H5ARG1RxevLO5smM/9LbXVAVgcZRnI3egmbaJF7r6queHtrQq5y27+Rz9OEoJ/DWMS/qaGWMqj1f7LDv+H44juKAxmayOYkLJUnXN/AYITjpNSfLy0kuSo9cGpHVKjQZVFGv92b0c1CPrQ0WJ3Q8Y7ERc77b2PBQmxWow/3RHv6SoeT8riCfoU5Mnuld2aNjlJlEglf79QB8r7Xonx3TDfcr0+bGbvJNhYoMPPDXFC96wCpjvIDQu09BEM91/zP/VRE3E+bvIp0WRugU6vUYAAIcKuyQpON5Fs1kWaY4AZ6/4+Qdql5B8z8twz +vhagar.oob.openstreetmap.org,vhagar.oob,10.0.49.5 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJaXI71RCPjbni+WxBnOf2CzkLi0H5ARG1RxevLO5smM/9LbXVAVgcZRnI3egmbaJF7r6queHtrQq5y27+Rz9OEoJ/DWMS/qaGWMqj1f7LDv+H44juKAxmayOYkLJUnXN/AYITjpNSfLy0kuSo9cGpHVKjQZVFGv92b0c1CPrQ0WJ3Q8Y7ERc77b2PBQmxWow/3RHv6SoeT8riCfoU5Mnuld2aNjlJlEglf79QB8r7Xonx3TDfcr0+bGbvJNhYoMPPDXFC96wCpjvIDQu09BEM91/zP/VRE3E+bvIp0WRugU6vUYAAIcKuyQpON5Fs1kWaY4AZ6/4+Qdql5B8z8twz dulcy.oob.openstreetmap.org,dulcy.oob,10.0.49.9 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgmlwXGPIPDqoMz4s5edr+G4iuBnCYcFXjY7Et9jG/ezI6aIKd9lEOXvnHJX5hrC21aqd06dhyMLwZN+eSjJhXNaLtYmm5P442H2ZnGca1KNSpF7yRVhn0eRpX39xWK3biVfMw64mzvV3636C1adCkgBwuaqCvz1EHm/KRrySfuETFLU= spike-06.oob.openstreetmap.org,spike-06.oob,10.0.49.6 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDwQVb/cYgKp21vw6+jKr5TLAD77oKunbFbjvwPL34iLRDqdJcEOdJBV1A63ZDUWSWjDUrTa0kyneoiYdD876EpuspZ4bPgmiHBvug5NR3usxa8PAsia1K+fONAGi3+s96H6Us273KHNH5QwyBNDyYaDDjIECkydU4bQT0FSX0j32eAHI62ZO/H82fUyL20PIK/nCUaJUd+5iXgcTfjP+eL66ghgPCgXGF99/w2+3EUxP2IN9J2yI57H/rYn3rj+NdZyQuzc0TAToT341cYh2+a8R1Sb/DEFiMZFE2hJ2NeVEaBYxPSlFfhRtlB6ArXg5QL7e7UScQh/ZHLU26A1ZaB spike-07.oob.openstreetmap.org,spike-07.oob,10.0.49.7 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxLxAbMPZw+LgzB8JmTLLxqc6CZXYA3Wo18RM02+EnmtRX6AAf/VFb7E/VpgHfZecwnFK7u21R47+fFdHCb8EDYGeWbOYoEOYMoDh26H8/aFPHbldgG0xs9EwfTryWB7iZ2sD9nLv+nBpDkFXVq53kscmDlUKQrvUyj/zR55xGxcq0ruu3w05ZeGx5I6HRY0xeK+6H4s6sZtF24Zm3CttQ5M7ADgWffE35ZwA9tEdioM3J3c5EcV4HMdhl7wame6pID9+FpDEtcts5z5sfW9Y0yx3FHrXCxRWxHRbyFG9/+4RNwKJBRZrdb+VSLlrYVUFLEfb1AhOxr8ZfSsJU1V0V diff --git a/cookbooks/overpass/templates/default/totp-filter.erb b/cookbooks/overpass/templates/default/totp-filter.erb index 8245f2ae3..78160cfe9 100644 --- a/cookbooks/overpass/templates/default/totp-filter.erb +++ b/cookbooks/overpass/templates/default/totp-filter.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "cgi" require "rotp" diff --git a/cookbooks/oxidized/recipes/default.rb b/cookbooks/oxidized/recipes/default.rb index fce3126d3..1840c2af3 100644 --- a/cookbooks/oxidized/recipes/default.rb +++ b/cookbooks/oxidized/recipes/default.rb @@ -130,9 +130,15 @@ git "/var/lib/oxidized/configs.git" do group "oxidized" end +bundle_config "/opt/oxidized/daemon" do + user "oxidized" + group "oxidized" + settings "deployment" => "true", + "build.rugged" => "--with-ssh" +end + bundle_install "/opt/oxidized/daemon" do action :nothing - options "--deployment" user "oxidized" group "oxidized" notifies :restart, "service[oxidized]" @@ -150,6 +156,7 @@ systemd_service "oxidized" do "OXIDIZED_LOGS" => "/var/log/oxidized" nice 10 sandbox :enable_network => true + restrict_address_families "AF_NETLINK" read_write_paths ["/run/oxidized", "/var/lib/oxidized", "/var/log/oxidized"] restart "on-failure" notifies :restart, "service[oxidized]" diff --git a/cookbooks/passenger/templates/default/passenger.conf.erb b/cookbooks/passenger/templates/default/passenger.conf.erb index 1212f7588..5533f8deb 100644 --- a/cookbooks/passenger/templates/default/passenger.conf.erb +++ b/cookbooks/passenger/templates/default/passenger.conf.erb @@ -2,8 +2,9 @@ PassengerRoot /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini - PassengerDefaultRuby /usr/bin/ruby + PassengerDefaultRuby <%= node[:ruby][:interpreter] %> PassengerRuby /usr/local/bin/passenger-ruby + PassengerPreloadBundler on PassengerMaxPoolSize <%= node[:passenger][:max_pool_size] %> PassengerPoolIdleTime <%= node[:passenger][:pool_idle_time] %> PassengerFriendlyErrorPages off diff --git a/cookbooks/passenger/templates/default/ruby.erb b/cookbooks/passenger/templates/default/ruby.erb index f45fd210e..2af922032 100644 --- a/cookbooks/passenger/templates/default/ruby.erb +++ b/cookbooks/passenger/templates/default/ruby.erb @@ -4,4 +4,4 @@ export RUBY_GC_HEAP_INIT_SLOTS=500000 export RUBY_GC_HEAP_FREE_SLOTS=100000 export RUBY_GC_MALLOC_LIMIT=50000000 -exec /usr/bin/ruby<%= node[:ruby][:version] %> "$@" +exec <%= node[:ruby][:interpreter] %> "$@" diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb index d719d11ae..f7552ddf2 100644 --- a/cookbooks/planet/recipes/replication.rb +++ b/cookbooks/planet/recipes/replication.rb @@ -32,14 +32,18 @@ db_passwords = data_bag_item("db", "passwords") package %w[ postgresql-client - ruby-libxml make gcc libc6-dev libpq-dev + libxml2-dev osmdbt ] +gem_package "libxml-ruby" do + gem_binary node[:ruby][:gem] +end + gem_package "pg" do gem_binary node[:ruby][:gem] end @@ -91,6 +95,13 @@ template "/usr/local/bin/users-deleted" do mode "755" end +template "/usr/local/bin/replicate-changesets" do + source "replicate-changesets.erb" + owner "root" + group "root" + mode "755" +end + ## Published deleted users directory remote_directory "/store/planet/users_deleted" do diff --git a/cookbooks/planet/templates/default/planet-file-cleanup.erb b/cookbooks/planet/templates/default/planet-file-cleanup.erb index 52ce68623..1df8eb595 100644 --- a/cookbooks/planet/templates/default/planet-file-cleanup.erb +++ b/cookbooks/planet/templates/default/planet-file-cleanup.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require 'date' require 'optparse' diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/templates/default/replicate-changesets.erb similarity index 90% rename from cookbooks/planet/files/default/replication-bin/replicate-changesets rename to cookbooks/planet/templates/default/replicate-changesets.erb index 0c53856d9..e2e5aca68 100755 --- a/cookbooks/planet/files/default/replication-bin/replicate-changesets +++ b/cookbooks/planet/templates/default/replicate-changesets.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "rubygems" require "pg" @@ -96,7 +96,7 @@ class ChangesetBuilder def add_comments(xml, cs) # grab the visible changeset comments as well - res = @conn.exec("select cc.author_id, u.display_name as author, cc.body, cc.created_at from changeset_comments cc join users u on cc.author_id=u.id where cc.changeset_id=#{cs.id} and cc.visible order by cc.created_at asc") + res = @conn.exec("select cc.id, cc.author_id, u.display_name as author, cc.body, (cc.created_at at time zone 'utc') as created_at from changeset_comments cc join users u on cc.author_id=u.id where cc.changeset_id=#{cs.id} and cc.visible order by cc.created_at asc") xml["comments_count"] = res.num_tuples.to_s # early return if there aren't any comments @@ -105,6 +105,7 @@ class ChangesetBuilder discussion = XML::Node.new("discussion") res.each do |row| comment = XML::Node.new("comment") + comment["id"] = row["id"] comment["uid"] = row["author_id"] comment["user"] = xml_sanitize(row["author"]) comment["date"] = Time.parse(row["created_at"]).getutc.xmlschema @@ -127,7 +128,7 @@ end # sync a directory to guarantee it's on disk. have to recurse to the root # to guarantee sync for newly created directories. def fdirsync(d) - while d != "/" + while d != "/" && d != "." fsync(d) d = File.dirname(d) end @@ -139,7 +140,7 @@ end class Replicator def initialize(config) @config = YAML.safe_load(File.read(config)) - @state = YAML.safe_load(File.read(@config["state_file"]), [Time]) + @state = YAML.safe_load(File.read(@config["state_file"]), :permitted_classes => [Time], :fallback => {}) @conn = PG::Connection.connect(@config["db"]) # get current time from the database rather than the current system @now = @conn.exec("select now() as now").map { |row| Time.parse(row["now"]) }[0] @@ -153,7 +154,7 @@ class Replicator # for us to look at anything that was closed recently, and filter from # there. changesets = @conn - .exec("select id, created_at, closed_at, num_changes from changesets where closed_at > ((now() at time zone 'utc') - '1 hour'::interval)") + .exec("select id, (created_at at time zone 'utc') as created_at, (closed_at at time zone 'utc') as closed_at, num_changes from changesets where (closed_at at time zone 'utc') > ((now() at time zone 'utc') - '1 hour'::interval)") .map { |row| Changeset.new(row) } .select { |cs| cs.activity_between?(last_run, @now) } @@ -162,13 +163,13 @@ class Replicator # but also add any changesets which have new comments new_ids = @conn - .exec("select distinct changeset_id from changeset_comments where created_at >= '#{last_run}' and created_at < '#{@now}' and visible") + .exec("select distinct changeset_id from changeset_comments where (created_at at time zone 'utc') >= '#{last_run}' and (created_at at time zone 'utc') < '#{@now}' and visible") .map { |row| row["changeset_id"].to_i } .reject { |c_id| cs_ids.include?(c_id) } new_ids.each do |id| @conn - .exec("select id, created_at, closed_at, num_changes from changesets where id=#{id}") + .exec("select id, (created_at at time zone 'utc') as created_at, (closed_at at time zone 'utc') as closed_at, num_changes from changesets where id=#{id}") .map { |row| Changeset.new(row) } .each { |cs| changesets << cs } end diff --git a/cookbooks/podman/resources/service.rb b/cookbooks/podman/resources/service.rb index 5ef0b5885..d9a328192 100644 --- a/cookbooks/podman/resources/service.rb +++ b/cookbooks/podman/resources/service.rb @@ -27,6 +27,7 @@ property :image, String, :required => true property :ports, Hash, :default => {} property :environment, Hash, :default => {} property :volume, Hash, :default => {} +property :command, String, :default => "" action :create do systemd_service new_resource.service do @@ -35,7 +36,11 @@ action :create do notify_access "all" environment "PODMAN_SYSTEMD_UNIT" => "%n" exec_start_pre "/bin/rm --force %t/%n.ctr-id" - exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --userns=auto --label=io.containers.autoupdate=registry --pids-limit=-1 #{publish_options} #{environment_options} #{volume_options} --rm --sdnotify=conmon --detach --replace --name=%N #{new_resource.image}" + exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon "\ + "--userns=auto --label=io.containers.autoupdate=registry "\ + "--pids-limit=-1 #{publish_options} #{environment_options} "\ + "#{volume_options} --rm --sdnotify=conmon --detach --replace "\ + "--name=%N #{new_resource.image} #{new_resource.command}" exec_stop "/usr/bin/podman stop --ignore --time=10 --cidfile=%t/%n.ctr-id" exec_stop_post "/usr/bin/podman rm --force --ignore --cidfile=%t/%n.ctr-id" timeout_start_sec 180 diff --git a/cookbooks/postgresql/recipes/default.rb b/cookbooks/postgresql/recipes/default.rb index 742941ee9..eae492f16 100644 --- a/cookbooks/postgresql/recipes/default.rb +++ b/cookbooks/postgresql/recipes/default.rb @@ -124,24 +124,20 @@ clusters.each do |name, details| not_if { ::File.exist?("/var/lib/postgresql/#{name}/standby.signal") } end - exporter_options = %w[ - --collector.database_wraparound - --collector.long_running_transactions - --collector.process_idle - --collector.stat_activity_autovacuum - --collector.stat_wal_receiver - --collector.statio_user_indexes - ] - - exporter_options << "--no-collector.stat_bgwriter" if details[:version] >= 17 - prometheus_exporter "postgres" do port 10000 + details[:port].to_i service "postgres-#{prometheus_suffix}" labels "cluster" => name scrape_interval "1m" scrape_timeout "1m" - options exporter_options + options %w[ + --collector.database_wraparound + --collector.long_running_transactions + --collector.process_idle + --collector.stat_activity_autovacuum + --collector.stat_wal_receiver + --collector.statio_user_indexes + ] environment "DATA_SOURCE_NAME" => "postgres:///#{prometheus_database}?host=/run/postgresql&port=#{details[:port]}&user=prometheus&password=#{passwords['prometheus']}" restrict_address_families "AF_UNIX" subscribes :restart, "template[/etc/prometheus/exporters/postgres_queries.yml]" diff --git a/cookbooks/prometheus/metadata.rb b/cookbooks/prometheus/metadata.rb index 30f20802d..6148979b9 100644 --- a/cookbooks/prometheus/metadata.rb +++ b/cookbooks/prometheus/metadata.rb @@ -12,4 +12,3 @@ depends "awscli" depends "git" depends "hardware" depends "networking" -depends "ruby" diff --git a/cookbooks/prometheus/recipes/default.rb b/cookbooks/prometheus/recipes/default.rb index c6b609b06..df3ec2ea5 100644 --- a/cookbooks/prometheus/recipes/default.rb +++ b/cookbooks/prometheus/recipes/default.rb @@ -19,7 +19,8 @@ include_recipe "git" include_recipe "networking" -include_recipe "ruby" + +package "ruby" if node.internal_ipaddress node.default[:prometheus][:mode] = "internal" diff --git a/cookbooks/prometheus/recipes/smokeping.rb b/cookbooks/prometheus/recipes/smokeping.rb index 27b0c6dd6..c626eb37c 100644 --- a/cookbooks/prometheus/recipes/smokeping.rb +++ b/cookbooks/prometheus/recipes/smokeping.rb @@ -39,6 +39,7 @@ end prometheus_exporter "smokeping" do port 9374 + environment "GOMAXPROCS" => "1" options "--config.file=/etc/prometheus/exporters/smokeping.yml" capability_bounding_set "CAP_NET_RAW" ambient_capabilities "CAP_NET_RAW" diff --git a/cookbooks/prometheus/resources/collector.rb b/cookbooks/prometheus/resources/collector.rb index ce68a7e5c..4171b874d 100644 --- a/cookbooks/prometheus/resources/collector.rb +++ b/cookbooks/prometheus/resources/collector.rb @@ -27,6 +27,7 @@ property :user, :kind_of => String property :path, :kind_of => String property :options, :kind_of => [String, Array] property :environment, :kind_of => Hash, :default => {} +property :working_directory, String property :proc_subset, String property :capability_bounding_set, [String, Array] property :private_devices, [true, false] @@ -42,6 +43,7 @@ action :create do dynamic_user new_resource.user.nil? group "adm" environment new_resource.environment + working_directory new_resource.working_directory standard_output "file:/var/lib/prometheus/node-exporter/#{new_resource.collector}.new" standard_error "journal" exec_start "#{executable_path} #{executable_options}" diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb index 305afbd90..27cb72e47 100644 --- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb +++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb @@ -828,7 +828,7 @@ groups: - name: web rules: - alert: web error rate - expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 and sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) > 0.01 + expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 and sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) > 0.05 for: 5m labels: alertgroup: web diff --git a/cookbooks/ruby/attributes/default.rb b/cookbooks/ruby/attributes/default.rb index eefaf2247..194f29957 100644 --- a/cookbooks/ruby/attributes/default.rb +++ b/cookbooks/ruby/attributes/default.rb @@ -1,9 +1,23 @@ -default[:ruby][:version] = if platform?("debian") - "3.1" - elsif node[:lsb][:release].to_f < 22.04 - "2.7" - else - "3.0" - end -default[:ruby][:gem] = "/usr/bin/gem#{node[:ruby][:version]}" -default[:ruby][:bundle] = "/usr/bin/bundle#{node[:ruby][:version]}" +default[:ruby][:fullstaq] = true + +if node[:ruby][:fullstaq] + + default[:ruby][:version] = "3.4" + default[:ruby][:interpreter] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/ruby" + default[:ruby][:gem] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/gem" + default[:ruby][:bundle] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/bundle" + +else + + default[:ruby][:version] = if platform?("debian") + "3.1" + elsif node[:lsb][:release].to_f < 22.04 + "2.7" + else + "3.0" + end + default[:ruby][:interpreter] = "/usr/bin/ruby#{node[:ruby][:version]}" + default[:ruby][:gem] = "/usr/bin/gem#{node[:ruby][:version]}" + default[:ruby][:bundle] = "/usr/bin/bundle#{node[:ruby][:version]}" + +end diff --git a/cookbooks/ruby/metadata.rb b/cookbooks/ruby/metadata.rb index d8f2eac0e..6c014bdfe 100644 --- a/cookbooks/ruby/metadata.rb +++ b/cookbooks/ruby/metadata.rb @@ -6,3 +6,4 @@ description "Installs and configures ruby" version "1.0.0" supports "ubuntu" +depends "apt" diff --git a/cookbooks/ruby/recipes/default.rb b/cookbooks/ruby/recipes/default.rb index a2833aefa..7301e372c 100644 --- a/cookbooks/ruby/recipes/default.rb +++ b/cookbooks/ruby/recipes/default.rb @@ -19,23 +19,29 @@ ruby_version = node[:ruby][:version] -package %W[ - ruby - ruby#{ruby_version} - ruby - ruby#{ruby_version}-dev -] - -gem_package "bundler#{ruby_version}-1" do - package_name "bundler" - version "~> 1.17.3" - gem_binary node[:ruby][:gem] - options "--format-executable" -end +if node[:ruby][:fullstaq] + + include_recipe "apt::fullstaq-ruby" + + package %W[ + fullstaq-ruby-common + fullstaq-ruby-#{ruby_version}-jemalloc + ] + + %w[bundle bundler erb gem irb racc rake rbs rdbg rdoc ri ruby syntax_suggest typeproc].each do |command| + link "/usr/local/bin/#{command}" do + to "/usr/lib/fullstaq-ruby/versions/#{ruby_version}-jemalloc/bin/#{command}" + owner "root" + group "root" + end + end + +else + + package %W[ + ruby + ruby-dev + ruby-bundler + ] -gem_package "bundler#{ruby_version}-2" do - package_name "bundler" - version "~> 2.3.16" - gem_binary node[:ruby][:gem] - options "--format-executable" end diff --git a/cookbooks/ruby/resources/bundle_config.rb b/cookbooks/ruby/resources/bundle_config.rb new file mode 100644 index 000000000..16902682f --- /dev/null +++ b/cookbooks/ruby/resources/bundle_config.rb @@ -0,0 +1,57 @@ +# +# Cookbook:: ruby +# Resource:: bundle_config +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +resource_name :bundle_config +provides :bundle_config + +unified_mode true + +default_action :create + +property :directory, :kind_of => String, :name_property => true +property :user, :kind_of => String +property :group, :kind_of => String +property :settings, :kind_of => Hash + +load_current_value do |new_resource| + current_settings = shell_out!("#{node[:ruby][:bundle]} config list --parseable", :cwd => new_resource.directory).stdout.split("\n").map do |line| + line.split("=") + end.to_h + + settings current_settings +end + +action :create do + converge_if_changed :settings do + new_resource.settings.each do |name, value| + execute "bundle-config-set-#{name}" do + command "#{bundle_command} config set --local #{name} #{value}" + cwd new_resource.directory + user new_resource.user + group new_resource.group + end + end + end +end + +action_class do + def bundle_command + node[:ruby][:bundle] + end +end diff --git a/cookbooks/ruby/resources/bundle_install.rb b/cookbooks/ruby/resources/bundle_install.rb index 897bde5fe..9facccc27 100644 --- a/cookbooks/ruby/resources/bundle_install.rb +++ b/cookbooks/ruby/resources/bundle_install.rb @@ -47,6 +47,9 @@ action_class do end def after_created - subscribes :run, "gem_package[bundler#{node[:ruby][:version]}-1]" - subscribes :run, "gem_package[bundler#{node[:ruby][:version]}-2]" + if node[:ruby][:fullstaq] + subscribes :run, "package[fullstaq-ruby-#{node[:ruby][:version]}-jemalloc]" + else + subscribes :run, "package[ruby-bundler]" + end end diff --git a/cookbooks/serverinfo/attributes/default.rb b/cookbooks/serverinfo/attributes/default.rb new file mode 100644 index 000000000..bd45555d1 --- /dev/null +++ b/cookbooks/serverinfo/attributes/default.rb @@ -0,0 +1 @@ +default[:accounts][:users][:serverinfo][:status] = :role diff --git a/cookbooks/serverinfo/metadata.rb b/cookbooks/serverinfo/metadata.rb index 55d69f066..48a196ad6 100644 --- a/cookbooks/serverinfo/metadata.rb +++ b/cookbooks/serverinfo/metadata.rb @@ -6,6 +6,7 @@ description "Configures server-info web site" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" depends "git" depends "ruby" diff --git a/cookbooks/serverinfo/recipes/default.rb b/cookbooks/serverinfo/recipes/default.rb index 2bbc31e5b..c455ddc12 100644 --- a/cookbooks/serverinfo/recipes/default.rb +++ b/cookbooks/serverinfo/recipes/default.rb @@ -17,6 +17,7 @@ # limitations under the License. # +include_recipe "accounts" include_recipe "apache" include_recipe "git" include_recipe "ruby" @@ -30,12 +31,18 @@ package %w[ pkg-config ] +directory "/srv/hardware.openstreetmap.org" do + owner "serverinfo" + group "serverinfo" + mode "755" +end + git "/srv/hardware.openstreetmap.org" do action :sync repository "https://github.com/osmfoundation/osmf-server-info.git" depth 1 - user "root" - group "root" + user "serverinfo" + group "serverinfo" notifies :run, "bundle_install[/srv/hardware.openstreetmap.org]" end @@ -45,52 +52,43 @@ roles = { :rows => search(:role, "*:*") } file "/srv/hardware.openstreetmap.org/_data/nodes.json" do content nodes.to_json mode "644" - owner "root" - group "root" + owner "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" + sensitive true end file "/srv/hardware.openstreetmap.org/_data/roles.json" do content roles.to_json mode "644" - owner "root" - group "root" + owner "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end -directory "/srv/hardware.openstreetmap.org/_site" do - mode "755" - owner "nobody" - group "nogroup" -end - -directory "/srv/hardware.openstreetmap.org/vendor" do +bundle_config "/srv/hardware.openstreetmap.org" do action :create - owner "nobody" - group "nogroup" - notifies :run, "bundle_install[/srv/hardware.openstreetmap.org]", :immediately + user "serverinfo" + group "serverinfo" + settings "deployment" => "true", + "without" => "development:test", + "jobs" => node.cpu_cores.to_s + notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end bundle_install "/srv/hardware.openstreetmap.org" do action :nothing - user "nobody" - group "nogroup" - environment "BUNDLE_FROZEN" => "true", - "BUNDLE_WITHOUT" => "development:test", - "BUNDLE_PATH" => "vendor/bundle", - "BUNDLE_DEPLOYMENT" => "1", - "BUNDLE_JOBS" => node.cpu_cores.to_s + user "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end bundle_exec "/srv/hardware.openstreetmap.org" do action :nothing command "jekyll build --trace --disable-disk-cache --baseurl=https://hardware.openstreetmap.org" - user "nobody" - group "nogroup" - environment "LANG" => "C.UTF-8", - "BUNDLE_PATH" => "vendor/bundle", - "BUNDLE_DEPLOYMENT" => "1" + user "serverinfo" + group "serverinfo" + environment "LANG" => "C.UTF-8" end ssl_certificate "hardware.openstreetmap.org" do diff --git a/cookbooks/stateofthemap/recipes/container.rb b/cookbooks/stateofthemap/recipes/container.rb index 3b5bd38b9..8e97accc0 100644 --- a/cookbooks/stateofthemap/recipes/container.rb +++ b/cookbooks/stateofthemap/recipes/container.rb @@ -24,7 +24,7 @@ podman_site "stateofthemap.org" do aliases ["www.stateofthemap.org", "stateofthemap.com", "www.stateofthemap.com", "sotm.org", "www.sotm.org"] end -%w[2007 2008 2009 2013 2016 2017 2018 2019 2020 2021 2022 2024].each do |year| +%w[2007 2008 2009 2013 2016 2017 2018 2019 2020 2021 2022 2024 2025].each do |year| podman_site "#{year}.stateofthemap.org" do image "ghcr.io/openstreetmap/stateofthemap-#{year}:latest" aliases ["#{year}.stateofthemap.com", "#{year}.sotm.org"] diff --git a/cookbooks/systemd/resources/service.rb b/cookbooks/systemd/resources/service.rb index 11ca0e28c..8b380c306 100644 --- a/cookbooks/systemd/resources/service.rb +++ b/cookbooks/systemd/resources/service.rb @@ -36,6 +36,7 @@ property :notify_access, String, :is => %w[none main exec all] property :limit_nofile, Integer property :limit_as, [Integer, String] property :limit_cpu, [Integer, String] +property :limit_core, [Integer, String] property :memory_low, [Integer, String] property :memory_high, [Integer, String] property :memory_max, [Integer, String] diff --git a/cookbooks/systemd/templates/default/service.erb b/cookbooks/systemd/templates/default/service.erb index 92d70de85..eee220691 100644 --- a/cookbooks/systemd/templates/default/service.erb +++ b/cookbooks/systemd/templates/default/service.erb @@ -45,6 +45,9 @@ LimitAS=<%= @limit_as %> <% if @limit_cpu -%> LimitCPU=<%= @limit_cpu %> <% end -%> +<% if @limit_core -%> +LimitCORE=<%= @limit_core %> +<% end -%> <% if @memory_low -%> MemoryLow=<%= @memory_low %> <% end -%> diff --git a/cookbooks/taginfo/recipes/default.rb b/cookbooks/taginfo/recipes/default.rb index 080d11924..ef3913c60 100644 --- a/cookbooks/taginfo/recipes/default.rb +++ b/cookbooks/taginfo/recipes/default.rb @@ -163,7 +163,7 @@ node[:taginfo][:sites].each do |site| settings["opensearch"]["contact"] = "webmaster@openstreetmap.org" settings["paths"]["bin_dir"] = "#{directory}/build/src" settings["sources"]["download"] = "" - settings["sources"]["create"] = "db languages projects wiki wikidata chronology" + settings["sources"]["create"] = "db languages projects wiki wikidata chronology sw" settings["sources"]["db"]["planetfile"] = "/var/lib/planet/planet.osh.pbf" settings["sources"]["chronology"]["osm_history_file"] = "/var/lib/planet/planet.osh.pbf" settings["tagstats"]["geodistribution"] = "DenseMmapArray" @@ -179,10 +179,17 @@ node[:taginfo][:sites].each do |site| notifies :restart, "service[apache2]" end + bundle_config "#{directory}/taginfo" do + user "taginfo" + group "taginfo" + settings "deployment" => "true", + "without" => "development:test" + end + bundle_install "#{directory}/taginfo" do action :nothing - user "root" - group "root" + user "taginfo" + group "taginfo" subscribes :run, "git[#{directory}/taginfo]" notifies :restart, "passenger_application[#{directory}/taginfo/web/public]" end @@ -225,7 +232,8 @@ node[:taginfo][:sites].each do |site| prometheus_collector "taginfo-#{site_name}" do interval "15m" user "taginfo" - path "#{directory}/taginfo/sources/metrics.rb" - options "#{directory}/data" + path node[:ruby][:bundle] + options "exec sources/metrics.rb #{directory}/data" + working_directory "#{directory}/taginfo" end end diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 13b897363..f969546af 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -159,6 +159,7 @@ end package %w[ python3-cairo python3-mapnik + python3-pil python3-pyotp python3-pyproj python3-setuptools diff --git a/cookbooks/tile/templates/default/expire-tiles.erb b/cookbooks/tile/templates/default/expire-tiles.erb index b3a790f6a..5bc478082 100644 --- a/cookbooks/tile/templates/default/expire-tiles.erb +++ b/cookbooks/tile/templates/default/expire-tiles.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> # DO NOT EDIT - This file is being maintained by Chef diff --git a/cookbooks/tile/templates/default/export.erb b/cookbooks/tile/templates/default/export.erb index 8fa4672b9..eb35b1313 100644 --- a/cookbooks/tile/templates/default/export.erb +++ b/cookbooks/tile/templates/default/export.erb @@ -5,6 +5,7 @@ import cairo import cgi import http.cookies import mapnik +import io import os import pyotp import pyproj @@ -13,6 +14,7 @@ import shutil import signal import sys import tempfile +from PIL import Image # Limit maximum CPU time # The Postscript output format can sometimes take hours @@ -40,6 +42,10 @@ def output_file(file): def file_size(file): return os.fstat(file.fileno()).st_size +# Routine to retrieve BytesIO payload length +def bytesio_size(bio): + return bio.getbuffer().nbytes + # Routine to report an error def output_error(message, status = "400 Bad Request"): print("Status: %s" % status) @@ -54,6 +60,105 @@ def output_error(message, status = "400 Bad Request"): print("") print("") +# Add a copyright notice for raster formats (PNG, JPEG, WEBP) +def add_copyright_notice_raster(image, map_width, map_height, format): + # Convert the Mapnik image to PNG and store it in a BytesIO object + png = image.tostring("png") + png_io = io.BytesIO(png) + + # Load the PNG data from the BytesIO object into a Cairo ImageSurface + surface = cairo.ImageSurface.create_from_png(png_io) + + add_copyright_notice_vector(surface, map_width, map_height) + + # Convert the Cairo surface to PNG in a BytesIO object + output_io = io.BytesIO() + surface.write_to_png(output_io) + + if format == "png": + return output_io + else: + # Open the output PNG image for conversion to other formats + img = Image.open(output_io) + img_io = io.BytesIO() + img.save(img_io, format=format) + return img_io + +# Add a copyright notice for vector formats (SVG, PDF, PS) +def add_copyright_notice_vector(surface, map_width, map_height): + context = cairo.Context(surface) + + # Set the font for the copyright notice + context.set_font_face(cairo.ToyFontFace("DejaVu")) + context.set_font_size(14) + + # Define the copyright text + text = "© OpenStreetMap contributors" + + text_extents = context.text_extents(text) + text_width = text_extents.width + text_height = text_extents.height + + x_margin = 10 + y_margin = 10 + + # Position the text at the bottom-right corner + x_position = map_width - text_width - x_margin + y_position = map_height - text_height - y_margin + + # Draw a white box just large enough to fit the text + context.set_source_rgba(1, 1, 1, 0.5) + context.rectangle(x_position - x_margin, y_position - y_margin, + text_width + 2 * x_margin, text_height + 2 * y_margin) + context.fill_preserve() + + context.set_source_rgb(0, 0, 0) # Black color for the text + context.move_to(x_position - x_margin / 2, y_position + y_margin) + context.show_text(text) + +# Render and output map for raster formats (PNG, JPEG, WEBP) +def render_and_output_image(map, format): + image = mapnik.Image(map.width, map.height) + mapnik.render(map, image) + + bytes_io = add_copyright_notice_raster(image, map.width, map.height, format) + + if format == "png": + output_headers("image/png", "map.png", bytesio_size(bytes_io)) + elif format == "jpeg": + output_headers("image/jpeg", "map.jpg", bytesio_size(bytes_io)) + elif format == "webp": + output_headers("image/webp", "map.webp", bytesio_size(bytes_io)) + + output_file(bytes_io) + +# Render and output map for vector formats (SVG, PDF, PS) +def render_and_output_vector(map, format): + with tempfile.NamedTemporaryFile(prefix="export") as file: + if format == "svg": + surface = cairo.SVGSurface(file.name, map.width, map.height) + surface.restrict_to_version(cairo.SVG_VERSION_1_2) + elif format == "pdf": + surface = cairo.PDFSurface(file.name, map.width, map.height) + elif format == "ps": + surface = cairo.PSSurface(file.name, map.width, map.height) + + mapnik.render(map, surface) + + add_copyright_notice_vector(surface, map.width, map.height) + + surface.finish() + + if format == "svg": + output_headers("image/svg+xml", "map.svg", file_size(file)) + elif format == "pdf": + output_headers("application/pdf", "map.pdf", file_size(file)) + elif format == "ps": + output_headers("application/postscript", "map.ps", file_size(file)) + + output_file(file) + + # Create TOTP token validator totp = pyotp.TOTP('<%= @totp_key %>', interval = 3600) @@ -149,42 +254,13 @@ else: # Render the map if pid == 0: - if form.getvalue("format") == "png": - image = mapnik.Image(map.width, map.height) - mapnik.render(map, image) - png = image.tostring("png") - output_headers("image/png", "map.png", len(png)) - sys.stdout.buffer.write(png) - elif form.getvalue("format") == "jpeg": - image = mapnik.Image(map.width, map.height) - mapnik.render(map, image) - jpeg = image.tostring("jpeg") - output_headers("image/jpeg", "map.jpg", len(jpeg)) - sys.stdout.buffer.write(jpeg) - elif form.getvalue("format") == "svg": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.SVGSurface(file.name, map.width, map.height) - surface.restrict_to_version(cairo.SVG_VERSION_1_2) - mapnik.render(map, surface) - surface.finish() - output_headers("image/svg+xml", "map.svg", file_size(file)) - output_file(file) - elif form.getvalue("format") == "pdf": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.PDFSurface(file.name, map.width, map.height) - mapnik.render(map, surface) - surface.finish() - output_headers("application/pdf", "map.pdf", file_size(file)) - output_file(file) - elif form.getvalue("format") == "ps": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.PSSurface(file.name, map.width, map.height) - mapnik.render(map, surface) - surface.finish() - output_headers("application/postscript", "map.ps", file_size(file)) - output_file(file) + format = form.getvalue("format") + if format in ["png", "jpeg", "webp"]: + render_and_output_image(map, format) + elif format in ["svg", "pdf", "ps"]: + render_and_output_vector(map, format) else: - output_error("Unknown format '%s'" % form.getvalue("format")) + output_error("Unknown format") else: pid, status = os.waitpid(pid, 0) if status & 0xff == signal.SIGXCPU: diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb index b5e0933a6..f72621213 100644 --- a/cookbooks/vectortile/attributes/default.rb +++ b/cookbooks/vectortile/attributes/default.rb @@ -4,10 +4,12 @@ default[:vectortile][:database][:nodes_store] = :flat default[:vectortile][:serve][:threads] = node.cpu_cores default[:vectortile][:serve][:mode] = :live default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute" -default[:vectortile][:replication][:status] = :enabled -default[:vectortile][:replication][:tileupdate] = :enabled +default[:vectortile][:replication][:enabled] = true +default[:vectortile][:replication][:tileupdate] = true default[:vectortile][:replication][:threads] = node.cpu_cores +default[:vectortile][:tilekiln][:version] = "0.7.1" + default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] default[:postgresql][:monitor_database] = "tiles" # As an absolute worst case, the server might have the serving, update, and a manual generation process going on. diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb index 48b60f3c8..0f4041565 100644 --- a/cookbooks/vectortile/recipes/default.rb +++ b/cookbooks/vectortile/recipes/default.rb @@ -86,7 +86,7 @@ shortbread_config = "#{style_directory}/shortbread.yaml" themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark" git themepark_directory do repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git" - revision "7efc00db433fa7d2c49db00c32ffd27d8ced35f4" + revision "444bfbda82dea2899e77ac7f0e88ddf7f62c3b45" user "tileupdate" group "tileupdate" end @@ -100,7 +100,7 @@ end python_package "tilekiln" do python_virtualenv tilekiln_directory python_version "3" - version "0.6.5" + version node[:vectortile][:tilekiln][:version] end template "/srv/vector.openstreetmap.org/html/index.html" do @@ -246,19 +246,21 @@ execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do end template "/usr/local/bin/vector-update" do - source node[:vectortile][:replication][:tileupdate] == :enabled ? "vector-update-tile.erb" : "vector-update-notile.erb" + source node[:vectortile][:replication][:tileupdate] ? "vector-update-tile.erb" : "vector-update-notile.erb" owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :post_processing => "/usr/local/bin/tiles-rerender" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender" end +rerender_layers = %w[addresses boundaries bridges buildings land pois public_transport sites street_polygons streets water_lines_labels water_lines water_polygons].join(" ") + template "/usr/local/bin/tiles-rerender" do source "tiles-rerender.erb" owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :update_threads => 4 + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4, :layers => "#{rerender_layers}" end systemd_service "replicate" do @@ -279,7 +281,7 @@ systemd_timer "replicate" do accuracy_sec 5 end -if node[:vectortile][:replication][:status] == :enabled +if node[:vectortile][:replication][:enabled] service "replicate.timer" do action [:enable, :start] end diff --git a/cookbooks/vectortile/templates/default/import-planet.erb b/cookbooks/vectortile/templates/default/import-planet.erb index bffa34a3a..3d6110181 100644 --- a/cookbooks/vectortile/templates/default/import-planet.erb +++ b/cookbooks/vectortile/templates/default/import-planet.erb @@ -7,7 +7,7 @@ set -e -export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' # Import the osm2pgsql file specified as an argument, using the locations for spirit osm2pgsql \ diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb index f46b6f1f6..d1ca4bdab 100644 --- a/cookbooks/vectortile/templates/default/tiles-rerender.erb +++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb @@ -1,7 +1,14 @@ #!/bin/sh set -eu -<%= @tilekiln_bin %> generate tiles \ + +cd "<%= @expiry_dir %>" +# Only expire z10+ tiles +rm -f z?-*.txt +wc -l z*.txt +(for layer in <%= @layers %>; do + find . -type f -name "z*-${layer}.txt" -execdir sed -e "s/\$/,${layer}/" {} \; +done) | <%= @tilekiln_bin %> generate layers \ --source-dbname "<%= @source_database %>" \ --storage-dbname "<%= @storage_database %>" \ --num-threads "<%= node[:vectortile][:replication][:threads] %>" \ ---config <%= @config_path %> < <%= @tiles_file %> +--config <%= @config_path %> diff --git a/cookbooks/vectortile/templates/default/vector-update-notile.erb b/cookbooks/vectortile/templates/default/vector-update-notile.erb index cd718789c..844d9d4fa 100644 --- a/cookbooks/vectortile/templates/default/vector-update-notile.erb +++ b/cookbooks/vectortile/templates/default/vector-update-notile.erb @@ -1,12 +1,13 @@ #!/bin/sh # Usage -# sudo -u tilekiln vector-update +# sudo -u tileupdate vector-update set -eu -export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' +cd "<%= @expiry_dir %>" osm2pgsql-replication update \ -d "<%= @source_database %>" \ --max-diff-size "<%= @diff_size %>" diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb index e59923b27..a356cbb10 100644 --- a/cookbooks/vectortile/templates/default/vector-update-tile.erb +++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb @@ -5,11 +5,12 @@ set -eu -export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' + +cd "<%= @expiry_dir %>" +rm -f z*.txt osm2pgsql-replication update \ -d "<%= @source_database %>" \ --max-diff-size "<%= @diff_size %>" \ - --post-processing "<%= @post_processing %>" \ - -- --expire-tiles=10-14 \ - --expire-output="<%= @tiles_file %>" + --post-processing "<%= @post_processing %>" diff --git a/cookbooks/web/recipes/statistics.rb b/cookbooks/web/recipes/statistics.rb index 84848b003..88c85a6fe 100644 --- a/cookbooks/web/recipes/statistics.rb +++ b/cookbooks/web/recipes/statistics.rb @@ -19,7 +19,6 @@ include_recipe "web::base" -ruby = "ruby#{node[:ruby][:version]}" rails_directory = "#{node[:web][:base_directory]}/rails" template "/usr/local/bin/statistics" do @@ -27,7 +26,7 @@ template "/usr/local/bin/statistics" do owner "root" group "root" mode "755" - variables :ruby => ruby, :directory => rails_directory + variables :ruby => node[:ruby][:interpreter], :directory => rails_directory end systemd_service "web-statistics" do diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb index debc4db50..681690452 100644 --- a/cookbooks/web/resources/rails_port.rb +++ b/cookbooks/web/resources/rails_port.rb @@ -408,11 +408,25 @@ action :create do action :delete end + directory "#{rails_directory}/vendor/bundle/ruby/#{node[:ruby][:version]}.0" do + owner new_resource.user + group new_resource.group + mode "775" + recursive true + end + + bundle_config "#{rails_directory}" do + user new_resource.user + group new_resource.group + settings "deployment" => "true", + "build.nokogiri" => "--use-system-libraries" + end + bundle_install "#{rails_directory}" do action :nothing - user "root" - group "root" - environment "NOKOGIRI_USE_SYSTEM_LIBRARIES" => "yes" + user new_resource.user + group new_resource.group + subscribes :run, "directory[#{rails_directory}/vendor/bundle/ruby/#{node[:ruby][:version]}.0]" subscribes :run, "git[#{rails_directory}]" end diff --git a/cookbooks/web/templates/default/apache.frontend.erb b/cookbooks/web/templates/default/apache.frontend.erb index 39f2a6007..98097d7d9 100644 --- a/cookbooks/web/templates/default/apache.frontend.erb +++ b/cookbooks/web/templates/default/apache.frontend.erb @@ -186,6 +186,7 @@ ErrorLog /var/log/apache2/error.log RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] diff --git a/cookbooks/web/templates/default/api-statistics.erb b/cookbooks/web/templates/default/api-statistics.erb index ce3af0b34..b6425ec88 100644 --- a/cookbooks/web/templates/default/api-statistics.erb +++ b/cookbooks/web/templates/default/api-statistics.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "apache_log_regex" require "file-tail" diff --git a/cookbooks/wiki/recipes/default.rb b/cookbooks/wiki/recipes/default.rb index e7a2b3e57..74e96d2cc 100644 --- a/cookbooks/wiki/recipes/default.rb +++ b/cookbooks/wiki/recipes/default.rb @@ -28,9 +28,7 @@ apache_site "default" do end mediawiki_site "wiki.openstreetmap.org" do - aliases ["wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmap.net", - "wiki.openstreetmap.ca", "wiki.openstreetmap.eu", - "wiki.openstreetmap.pro", "wiki.openstreetmaps.org", + aliases ["wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", "osm.wiki", "www.osm.wiki", "wiki.osm.wiki"] fpm_max_children 200 @@ -153,6 +151,7 @@ systemd_service "wiki-dump" do exec_start "/usr/bin/php w/maintenance/dumpBackup.php --full --quiet --output=gzip:dump/dump.xml.gz" working_directory "/srv/wiki.openstreetmap.org" user "wiki" + nice 19 sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" @@ -161,7 +160,7 @@ end systemd_timer "wiki-dump" do description "Wiki dump" - on_calendar "02:00" + on_calendar "Sun 02:30" end service "wiki-dump.timer" do diff --git a/roles/aarnet.rb b/roles/aarnet.rb deleted file mode 100644 index 8cab0ea35..000000000 --- a/roles/aarnet.rb +++ /dev/null @@ -1,27 +0,0 @@ -name "aarnet" -description "Role applied to all servers at AARNet" - -default_attributes( - :accounts => { - :users => { - :chm => { :status => :administrator }, - :bclifford => { :status => :administrator } - } - }, - :hosted_by => "AARNet", - :location => "Carlton, Victoria, Australia", - :timezone => "Australia/Melbourne" -) - -override_attributes( - :networking => { - :nameservers => ["202.158.207.1", "202.158.207.2"] - }, - :ntp => { - :servers => ["0.au.pool.ntp.org", "1.au.pool.ntp.org", "oceania.pool.ntp.org"] - } -) - -run_list( - "role[au]" -) diff --git a/roles/balerion.rb b/roles/balerion.rb deleted file mode 100644 index 4aef19a05..000000000 --- a/roles/balerion.rb +++ /dev/null @@ -1,60 +0,0 @@ -name "balerion" -description "Master role applied to balerion" - -default_attributes( - :networking => { - :interfaces => { - :external => { - :interface => "bond0", - :role => :external, - :inet => { - :address => "138.44.68.134", - :prefix => "30", - :gateway => "138.44.68.133" - }, - :bond => { - :slaves => %w[ens14f0np0 ens14f1np1] - } - } - } - }, - :postgresql => { - :settings => { - :defaults => { - :effective_cache_size => "16GB" - } - } - }, - :sysctl => { - :postgres => { - :comment => "Increase shared memory for postgres", - :parameters => { - "kernel.shmmax" => 9 * 1024 * 1024 * 1024, - "kernel.shmall" => 9 * 1024 * 1024 * 1024 / 4096 - } - } - }, - :tile => { - :database => { - :cluster => "16/main", - :postgis => "3" - }, - :mapnik => "3.1", - :replication => { - :directory => "/store/replication" - }, - :styles => { - :default => { - :tile_directories => [ - { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 } - ] - } - } - } -) - -run_list( - "role[aarnet]", - "role[geodns]", - "role[tile]" -) diff --git a/roles/base.rb b/roles/base.rb index 2a4f732a5..ee9b003c6 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -11,10 +11,6 @@ default_attributes( } }, :networking => { - :roles => { - :internal => { :metric => 200 }, - :external => { :metric => 100 } - }, :search => ["openstreetmap.org"] }, :prometheus => { diff --git a/roles/blogs.rb b/roles/blogs.rb index 3881d1396..87c71902a 100644 --- a/roles/blogs.rb +++ b/roles/blogs.rb @@ -1,6 +1,12 @@ name "blogs" description "Role applied to all blog aggregators" +default_attributes( + :ruby => { + :version => "3.3" + } +) + run_list( "recipe[blogs]" ) diff --git a/roles/bowser.rb b/roles/bowser.rb deleted file mode 100644 index eca0fa2cf..000000000 --- a/roles/bowser.rb +++ /dev/null @@ -1,59 +0,0 @@ -name "bowser" -description "Master role applied to bowser" - -default_attributes( - :networking => { - :interfaces => { - :external => { - :interface => "bond0", - :role => :external, - :inet => { - :address => "138.44.68.106", - :prefix => "30", - :gateway => "138.44.68.105" - }, - :bond => { - :slaves => %w[ens14f0np0 ens14f1np1] - } - } - } - }, - :postgresql => { - :settings => { - :defaults => { - :effective_cache_size => "16GB" - } - } - }, - :sysctl => { - :postgres => { - :comment => "Increase shared memory for postgres", - :parameters => { - "kernel.shmmax" => 9 * 1024 * 1024 * 1024, - "kernel.shmall" => 9 * 1024 * 1024 * 1024 / 4096 - } - } - }, - :tile => { - :database => { - :cluster => "16/main", - :postgis => "3" - }, - :mapnik => "3.1", - :replication => { - :directory => "/store/replication" - }, - :styles => { - :default => { - :tile_directories => [ - { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 } - ] - } - } - } -) - -run_list( - "role[aarnet]", - "role[tile]" -) diff --git a/roles/bytemark.rb b/roles/bytemark.rb index 8f23bd5d0..109a17208 100644 --- a/roles/bytemark.rb +++ b/roles/bytemark.rb @@ -3,30 +3,7 @@ description "Role applied to all servers at Bytemark" default_attributes( :hosted_by => "Bytemark", - :location => "York, England", - :networking => { - :roles => { - :internal => { - :inet => { - :prefix => "20", - :gateway => "10.0.32.20", - :routes => { - "10.0.0.0/8" => { :via => "10.0.32.20" } - } - } - }, - :external => { - :inet => { - :prefix => "28", - :gateway => "89.16.162.17" - }, - :inet6 => { - :prefix => "64", - :gateway => "fe80::1" - } - } - } - } + :location => "York, England" ) override_attributes( diff --git a/roles/culebre.rb b/roles/culebre.rb index d798bddae..8f266566a 100644 --- a/roles/culebre.rb +++ b/roles/culebre.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.9" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp68s0f0 enp68s0f1 enp68s0f2 enp68s0f3] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.105", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.105" }, :inet6 => { - :address => "2001:470:1:b3b::9", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::9" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.105", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.105" }, :inet6 => { - :address => "2001:4d78:fe03:1c::9", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::9" } } } @@ -83,6 +63,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[tile]" ) diff --git a/roles/dev.rb b/roles/dev.rb index f85849e88..432fe0d1c 100644 --- a/roles/dev.rb +++ b/roles/dev.rb @@ -116,7 +116,7 @@ default_attributes( :revision => "master", :cgimap_repository => "https://github.com/zerebubuth/openstreetmap-cgimap.git", :cgimap_revision => "master", - :aliases => ["api06.dev.openstreetmap.org"] + :aliases => ["master.apis.dev.osm.org", "api06.dev.openstreetmap.org", "api06.dev.osm.org"] }, :tomh => { :repository => "https://github.com/tomhughes/openstreetmap-website.git", @@ -159,6 +159,9 @@ default_attributes( } } }, + :ruby => { + :fullstaq => true + }, :sysctl => { :postgres => { :comment => "Increase shared memory for postgres", diff --git a/roles/dribble.rb b/roles/dribble.rb index 33707cfa4..b0bba8021 100644 --- a/roles/dribble.rb +++ b/roles/dribble.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.4" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.132", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.132" }, :inet6 => { - :address => "2001:470:1:fa1::4", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::4" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.100", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.100" }, :inet6 => { - :address => "2001:4d78:500:5e3::4", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::4" } } } @@ -60,6 +40,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[vectortile]" ) diff --git a/roles/dulcy.rb b/roles/dulcy.rb index d9d8fb79f..cc6b24fe6 100644 --- a/roles/dulcy.rb +++ b/roles/dulcy.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.9" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp1s0f0 enp1s0f1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.137", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.137" }, :inet6 => { - :address => "2001:470:1:fa1::9", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::9" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.105", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.105" }, :inet6 => { - :address => "2001:4d78:500:5e3::9", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::9" } } } @@ -62,13 +42,13 @@ default_attributes( :nominatim => { :dbcluster => "17/main", :flatnode_file => "/srv/nominatim.openstreetmap.org/planet-project/nodes.store", - :enable_qa_tiles => true, + :enable_qa_tiles => false, :api_workers => 14, :api_pool_size => 10 } ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[nominatim]" ) diff --git a/roles/eddie.rb b/roles/eddie.rb index 61e452455..2acf524e1 100644 --- a/roles/eddie.rb +++ b/roles/eddie.rb @@ -6,7 +6,6 @@ default_attributes( :interfaces => { :internal => { :interface => "enp1s0f0.2801", - :role => :internal, :inet => { :address => "10.0.0.10" } diff --git a/roles/equinix-ams-public.rb b/roles/equinix-ams-public.rb new file mode 100644 index 000000000..b89ae21bb --- /dev/null +++ b/roles/equinix-ams-public.rb @@ -0,0 +1,44 @@ +name "equinix-ams-public" +description "Role applied to all public servers at Equinix Amsterdam" + +default_attributes( + :networking => { + :interfaces => { + :henet => { + :interface => "bond0.3", + :role => :external, + :metric => 150, + :source_route_table => 3, + :inet => { + :prefix => "27", + :gateway => "184.104.179.129" + }, + :inet6 => { + :prefix => "64", + :gateway => "2001:470:1:fa1::1", + :routes => { + "2600:9000::/28" => { :table => 3, :type => "unreachable" } + } + } + }, + :equinix => { + :interface => "bond0.103", + :role => :external, + :metric => 100, + :source_route_table => 103, + :inet => { + :prefix => "27", + :gateway => "82.199.86.97" + }, + :inet6 => { + :prefix => "64", + :gateway => "2001:4d78:500:5e3::1" + } + } + } + } +) + +run_list( + "role[equinix-ams]" +) diff --git a/roles/equinix-ams.rb b/roles/equinix-ams.rb index ac8c6a885..c63522bbc 100644 --- a/roles/equinix-ams.rb +++ b/roles/equinix-ams.rb @@ -3,18 +3,23 @@ description "Role applied to all servers at Equinix Amsterdam" default_attributes( :networking => { - :roles => { + :interfaces => { :internal => { + :interface => "bond0", + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.48.14", :routes => { "10.0.0.0/8" => { :via => "10.0.48.14" } } + }, + :bond => { + :mode => "802.3ad", + :lacprate => "fast", + :xmithashpolicy => "layer3+4" } - }, - :external => { - :zone => "ams" } } }, diff --git a/roles/equinix-dub-public.rb b/roles/equinix-dub-public.rb new file mode 100644 index 000000000..343dfb4f9 --- /dev/null +++ b/roles/equinix-dub-public.rb @@ -0,0 +1,44 @@ +name "equinix-dub-public" +description "Role applied to all public servers at Equinix Dublin" + +default_attributes( + :networking => { + :interfaces => { + :henet => { + :interface => "bond0.101", + :role => :external, + :metric => 150, + :source_route_table => 101, + :inet => { + :prefix => "27", + :gateway => "184.104.226.97" + }, + :inet6 => { + :prefix => 64, + :gateway => "2001:470:1:b3b::1", + :routes => { + "2600:9000::/28" => { :table => 101, :type => "unreachable" } + } + } + }, + :equinix => { + :interface => "bond0.203", + :role => :external, + :metric => 100, + :source_route_table => 203, + :inet => { + :prefix => "27", + :gateway => "87.252.214.97" + }, + :inet6 => { + :prefix => 64, + :gateway => "2001:4d78:fe03:1c::1" + } + } + } + } +) + +run_list( + "role[equinix-dub]" +) diff --git a/roles/equinix-dub.rb b/roles/equinix-dub.rb index 6561d3019..741c09ab7 100644 --- a/roles/equinix-dub.rb +++ b/roles/equinix-dub.rb @@ -12,22 +12,27 @@ default_attributes( } }, :networking => { - :roles => { + :interfaces => { :internal => { + :interface => "bond0", + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.64.2", :routes => { "10.0.0.0/8" => { :via => "10.0.64.2" } - } - } - }, - :external => { - :zone => "dub", - :inet6 => { + }, :rules => [ - { :to => "2600:9000::/28", :table => 150, :priority => 100 } + { :to => "10.0.0.0/8", :table => "main", :priority => 50 }, + { :to => "172.16.0.0/12", :table => "main", :priority => 50 }, + { :to => "192.168.0.0/16", :table => "main", :priority => 50 } ] + }, + :bond => { + :mode => "802.3ad", + :lacprate => "fast", + :xmithashpolicy => "layer3+4" } } } diff --git a/roles/faffy.rb b/roles/faffy.rb index 34b480823..e02510d18 100644 --- a/roles/faffy.rb +++ b/roles/faffy.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.3" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.131", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.131" }, :inet6 => { - :address => "2001:470:1:fa1::3", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::3" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.99", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.99" }, :inet6 => { - :address => "2001:4d78:500:5e3::3", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::3" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[dev]" ) diff --git a/roles/fafnir.rb b/roles/fafnir.rb index bb6b7b667..e7edc263c 100644 --- a/roles/fafnir.rb +++ b/roles/fafnir.rb @@ -7,6 +7,7 @@ default_attributes( :last_address => "10.0.79.254" }, :exim => { + :external_interface => "<;${if <{${randint:100}}{75} {184.104.226.98;2001:470:1:b3b::2}{87.252.214.98;2001:4d78:fe03:1c::2}}", :routes => { :openstreetmap => { :comment => "openstreetmap.org", @@ -18,47 +19,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.2" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.98", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.98" }, :inet6 => { - :address => "2001:470:1:b3b::2", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::2" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.98", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.98" }, :inet6 => { - :address => "2001:4d78:fe03:1c::2", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::2" } } } @@ -81,7 +62,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[gateway]", "role[mail]", diff --git a/roles/firefishynet.rb b/roles/firefishynet.rb index 150f62d6f..bb4224f70 100644 --- a/roles/firefishynet.rb +++ b/roles/firefishynet.rb @@ -3,7 +3,7 @@ description "Role applied to all servers at Firefishy" default_attributes( :networking => { - :roles => { + :interfaces => { :internal => { :inet => { :prefix => "24", diff --git a/roles/fume.rb b/roles/fume.rb index 5cc06a4b1..af951e1f0 100644 --- a/roles/fume.rb +++ b/roles/fume.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.16" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.112", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.112" }, :inet6 => { - :address => "2001:470:1:b3b::10", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::10" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.112", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.112" }, :inet6 => { - :address => "2001:4d78:fe03:1c::10", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::10" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[community]" ) diff --git a/roles/geodns.rb b/roles/geodns.rb index a2ad092f3..a1df01532 100644 --- a/roles/geodns.rb +++ b/roles/geodns.rb @@ -12,8 +12,10 @@ default_attributes( :list => false, :transfer_logging => false, :hosts_allow => [ - "184.104.226.102", # idris - "2001:470:1:b3b::6" # idris + "184.104.226.102", # idris HE + "2001:470:1:b3b::6", # idris HE + "87.252.214.102", # idris Equinix + "2001:4d78:fe03:1c::6" # idris Equinix ] } } diff --git a/roles/grisu.rb b/roles/grisu.rb index 525e028d2..e7fd2f899 100644 --- a/roles/grisu.rb +++ b/roles/grisu.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.17" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.113", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.113" }, :inet6 => { - :address => "2001:470:1:b3b::11", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::11" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.113", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.113" }, :inet6 => { - :address => "2001:4d78:fe03:1c::11", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::11" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[overpass-query]" ) diff --git a/roles/horntail.rb b/roles/horntail.rb index d071856f4..b09bf796b 100644 --- a/roles/horntail.rb +++ b/roles/horntail.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.10" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0 enp25s0f1] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.106", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.106" }, :inet6 => { - :address => "2001:470:1:b3b::a", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::a" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.106", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.106" }, :inet6 => { - :address => "2001:4d78:fe03:1c::a", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::a" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[geodns]", "role[planet]" ) diff --git a/roles/idris.rb b/roles/idris.rb index 7467f67af..2a91aaf73 100644 --- a/roles/idris.rb +++ b/roles/idris.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.6" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.102", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.102" }, :inet6 => { - :address => "2001:470:1:b3b::6", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::6" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.102", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.102" }, :inet6 => { - :address => "2001:4d78:fe03:1c::6", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::6" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[chef-server]", "role[chef-repository]", diff --git a/roles/imagery.rb b/roles/imagery.rb index d6caa11e9..4a4a033a3 100644 --- a/roles/imagery.rb +++ b/roles/imagery.rb @@ -20,21 +20,13 @@ default_attributes( "net.core.somaxconn" => 10000 } } - }, - :nginx => { - :cache => { - :fastcgi => { - :enable => true, - :keys_zone => "fastcgi_cache_zone:256M", - :inactive => "45d", - :max_size => "51200M" - } - } } ) run_list( "recipe[imagery::au_agri]", + "recipe[imagery::au_act_aerial]", + "recipe[imagery::au_vic_melbourne_aerial]", "recipe[imagery::gb_ea]", "recipe[imagery::gb_hampshire_aerial]", "recipe[imagery::gb_os_sv]", diff --git a/roles/karm.rb b/roles/karm.rb index ac28c48d4..bb736d72c 100644 --- a/roles/karm.rb +++ b/roles/karm.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.50" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp1s0f0 enp1s0f1 enp2s0f0 enp2s0f1] } } diff --git a/roles/konqi.rb b/roles/konqi.rb index 4b1fbcea6..2ef8ebd84 100644 --- a/roles/konqi.rb +++ b/roles/konqi.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.7" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.103", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.103" }, :inet6 => { - :address => "2001:470:1:b3b::7", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::7" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.103", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.103" }, :inet6 => { - :address => "2001:4d78:fe03:1c::7", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::7" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[wiki]" ) diff --git a/roles/lockheed.rb b/roles/lockheed.rb index 09eddcc8b..82acb48b4 100644 --- a/roles/lockheed.rb +++ b/roles/lockheed.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.16" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno49 eno50] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.144", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.144" }, :inet6 => { - :address => "2001:470:1:fa1::10", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::10" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.112", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.112" }, :inet6 => { - :address => "2001:4d78:500:5e3::10", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::10" } } } @@ -85,18 +65,24 @@ default_attributes( }, :nginx => { :cache => { + :fastcgi => { + :enable => true, + :keys_zone => "fastcgi_cache_zone:2048M", + :inactive => "180d", + :max_size => "196608M" + }, :proxy => { :enable => true, - :keys_zone => "proxy_cache_zone:256M", + :keys_zone => "proxy_cache_zone:2048M", :inactive => "180d", - :max_size => "51200M" + :max_size => "196608M" } } } ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[imagery]" ) diff --git a/roles/longma.rb b/roles/longma.rb index b8bf00af1..b64af9aa4 100644 --- a/roles/longma.rb +++ b/roles/longma.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.13" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp68s0f0np0 enp68s0f1np1 enp68s0f2np2 enp68s0f3np3] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.109", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.109" }, :inet6 => { - :address => "2001:470:1:b3b::d", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::d" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.109", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.109" }, :inet6 => { - :address => "2001:4d78:fe03:1c::d", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::d" } } } @@ -70,6 +50,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[nominatim]" ) diff --git a/roles/muirdris.rb b/roles/muirdris.rb index b77e4f8df..a2c578567 100644 --- a/roles/muirdris.rb +++ b/roles/muirdris.rb @@ -8,47 +8,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.15" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.111", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.111" }, :inet6 => { - :address => "2001:470:1:b3b::f", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::f" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.111", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.111" }, :inet6 => { - :address => "2001:4d78:fe03:1c::f", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::f" } } } @@ -56,6 +36,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[gps-tile]" ) diff --git a/roles/naga.rb b/roles/naga.rb index d212b6680..1bcf3c176 100644 --- a/roles/naga.rb +++ b/roles/naga.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.8" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.104", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.104" }, :inet6 => { - :address => "2001:470:1:b3b::8", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::8" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.101", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.104" }, :inet6 => { - :address => "2001:4d78:fe03:1c::8", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::8" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[subversion]", "role[trac]", diff --git a/roles/norbert.rb b/roles/norbert.rb index ef3d33dbd..357dc0d5a 100644 --- a/roles/norbert.rb +++ b/roles/norbert.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.17" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0 enp25s0f1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.145", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.145" }, :inet6 => { - :address => "2001:470:1:fa1::11", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::11" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.113", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.113" }, :inet6 => { - :address => "2001:4d78:500:5e3::11", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::11" } } } @@ -56,7 +36,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[geodns]", "role[backup]", "role[planet]", diff --git a/roles/odin.rb b/roles/odin.rb index 1544fb9b3..9d5bd1cfc 100644 --- a/roles/odin.rb +++ b/roles/odin.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.15" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1np0 eno2np1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.143", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.143" }, :inet6 => { - :address => "2001:470:1:fa1::f", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::f" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.111", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.111" }, :inet6 => { - :address => "2001:4d78:500:5e3::f", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::f" } } } @@ -145,6 +125,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[tile]" ) diff --git a/roles/osuosl.rb b/roles/osuosl.rb index 99a3a992f..83fd46a72 100644 --- a/roles/osuosl.rb +++ b/roles/osuosl.rb @@ -11,8 +11,10 @@ default_attributes( :location => "Corvallis, Oregon", :timezone => "US/Pacific", :networking => { - :roles => { + :interfaces => { :external => { + :interface => "bond0", + :role => :external, :inet => { :prefix => "28", :gateway => "140.211.167.97" diff --git a/roles/ovh.rb b/roles/ovh.rb deleted file mode 100644 index f8391a2aa..000000000 --- a/roles/ovh.rb +++ /dev/null @@ -1,20 +0,0 @@ -name "ovh" -description "Role applied to all servers at OVH" - -default_attributes( - :hosted_by => "OVH", - :location => "Roubaix, France" -) - -override_attributes( - :networking => { - :nameservers => ["213.186.33.99"] - }, - :ntp => { - :servers => ["0.fr.pool.ntp.org", "1.fr.pool.ntp.org", "europe.pool.ntp.org"] - } -) - -run_list( - "role[fr]" -) diff --git a/roles/piasa.rb b/roles/piasa.rb index b1e775ad4..085fdc66e 100644 --- a/roles/piasa.rb +++ b/roles/piasa.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.101" }, diff --git a/roles/ridley.rb b/roles/ridley.rb index 0e4a66522..2a6b166c1 100644 --- a/roles/ridley.rb +++ b/roles/ridley.rb @@ -20,14 +20,12 @@ default_attributes( :interfaces => { :external => { :interface => "eth0.2800", - :role => :external, :inet => { :address => "193.60.236.19" } }, :internal => { :interface => "eth0.2801", - :role => :internal, :inet => { :address => "10.0.0.3" } @@ -37,7 +35,7 @@ default_attributes( ) run_list( - "role[ucl]", + "role[ucl-public]", "role[hp-dl360-g6]", "role[gateway]", "role[foundation]", diff --git a/roles/scorch.rb b/roles/scorch.rb deleted file mode 100644 index 0649374f4..000000000 --- a/roles/scorch.rb +++ /dev/null @@ -1,40 +0,0 @@ -name "scorch" -description "Master role applied to scorch" - -default_attributes( - :devices => { - :ssd_system => { - :comment => "Tune scheduler for system disk", - :type => "block", - :bus => "scsi", - :serial => "3600605b009bbf5601fc3206407a43546", - :attrs => { - "queue/scheduler" => "noop", - "queue/nr_requests" => "256", - "queue/read_ahead_kb" => "2048" - } - } - }, - :networking => { - :interfaces => { - :external => { - :interface => "eth0", - :role => :external, - :inet => { - :address => "176.31.235.79", - :prefix => "24", - :gateway => "176.31.235.254" - }, - :inet6 => { - :address => "2001:41d0:2:fc4f::1", - :prefix => "64", - :gateway => "2001:41d0:2:fcff:ff:ff:ff:ff" - } - } - } - } -) - -run_list( - "role[ovh]" -) diff --git a/roles/shenron.rb b/roles/shenron.rb index f91677226..7f9f36a1c 100644 --- a/roles/shenron.rb +++ b/roles/shenron.rb @@ -19,14 +19,6 @@ default_attributes( "it87" ] }, - :prometheus => { - :metrics => { - :exim_queue_limit => { :metric => 250 } - } - } -) - -override_attributes( :networking => { :dnssec => "false", :interfaces => { @@ -45,6 +37,11 @@ override_attributes( } } } + }, + :prometheus => { + :metrics => { + :exim_queue_limit => { :metric => 250 } + } } ) diff --git a/roles/smaug.rb b/roles/smaug.rb index 4156fcb99..7d42c4f6e 100644 --- a/roles/smaug.rb +++ b/roles/smaug.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.14" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.110", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.110" }, :inet6 => { - :address => "2001:470:1:b3b::e", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::e" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.110", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.110" }, :inet6 => { - :address => "2001:4d78:fe03:1c::e", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::e" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[matomo]" ) diff --git a/roles/snap-01.rb b/roles/snap-01.rb index d7fb452a5..e741e2d51 100644 --- a/roles/snap-01.rb +++ b/roles/snap-01.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.49" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4] } } diff --git a/roles/snap-02.rb b/roles/snap-02.rb index 80a2b3598..2a63385b3 100644 --- a/roles/snap-02.rb +++ b/roles/snap-02.rb @@ -6,7 +6,6 @@ default_attributes( :interfaces => { :internal => { :interface => "eno1.2801", - :role => :internal, :inet => { :address => "10.0.0.4" } diff --git a/roles/snap-03.rb b/roles/snap-03.rb index 4956bdab7..0164f8563 100644 --- a/roles/snap-03.rb +++ b/roles/snap-03.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.50" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0np0 enp25s0f1np1] } } diff --git a/roles/spike-01.rb b/roles/spike-01.rb index 80b56c5e1..51ff42c63 100644 --- a/roles/spike-01.rb +++ b/roles/spike-01.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.3" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.99", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.99" }, :inet6 => { - :address => "2001:470:1:b3b::3", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::3" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.99", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.99" }, :inet6 => { - :address => "2001:4d78:fe03:1c::3", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::3" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-02.rb b/roles/spike-02.rb index 065be3faf..4e178bdec 100644 --- a/roles/spike-02.rb +++ b/roles/spike-02.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.4" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.100", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.100" }, :inet6 => { - :address => "2001:470:1:b3b::4", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::4" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.100", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.100" }, :inet6 => { - :address => "2001:4d78:fe03:1c::4", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::4" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-03.rb b/roles/spike-03.rb index b620aecfc..a24512eaa 100644 --- a/roles/spike-03.rb +++ b/roles/spike-03.rb @@ -5,47 +5,28 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", :role => :internal, :inet => { :address => "10.0.64.5" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.101", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.101" }, :inet6 => { - :address => "2001:470:1:b3b::5", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::5" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.101", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.101" }, :inet6 => { - :address => "2001:4d78:fe03:1c::5", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::5" } } } @@ -53,7 +34,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-06.rb b/roles/spike-06.rb index dd754a7ba..b5d84e6a6 100644 --- a/roles/spike-06.rb +++ b/roles/spike-06.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.6" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.134", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.134" }, :inet6 => { - :address => "2001:470:1:fa1::6", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::6" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.102", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.102" }, :inet6 => { - :address => "2001:4d78:500:5e3::6", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::6" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]", "role[web-statistics]", diff --git a/roles/spike-07.rb b/roles/spike-07.rb index f9d0d8f12..166432919 100644 --- a/roles/spike-07.rb +++ b/roles/spike-07.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.7" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.135", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.135" }, :inet6 => { - :address => "2001:470:1:fa1::7", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::7" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.103", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.103" }, :inet6 => { - :address => "2001:4d78:500:5e3::7", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::7" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-08.rb b/roles/spike-08.rb index c46260340..46f0be788 100644 --- a/roles/spike-08.rb +++ b/roles/spike-08.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.8" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.136", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.136" }, :inet6 => { - :address => "2001:470:1:fa1::8", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::8" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.104", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.104" }, :inet6 => { - :address => "2001:4d78:500:5e3::8", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::8" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/stormfly-03.rb b/roles/stormfly-03.rb index d81b06c9f..374d2ab90 100644 --- a/roles/stormfly-03.rb +++ b/roles/stormfly-03.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.99" }, diff --git a/roles/stormfly-04.rb b/roles/stormfly-04.rb index 3a7d1a7c3..bcc11518a 100644 --- a/roles/stormfly-04.rb +++ b/roles/stormfly-04.rb @@ -8,8 +8,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.100" }, diff --git a/roles/tabaluga.rb b/roles/tabaluga.rb index af8492b83..1ad1ee227 100644 --- a/roles/tabaluga.rb +++ b/roles/tabaluga.rb @@ -9,47 +9,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.14" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.142", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.142" }, :inet6 => { - :address => "2001:470:1:fa1::e", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::e" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.110", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.110" }, :inet6 => { - :address => "2001:4d78:500:5e3::e", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::e" } } } @@ -72,7 +52,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[blog-staging]", "role[taginfo]", diff --git a/roles/taginfo.rb b/roles/taginfo.rb index 40bffdce2..9b8227957 100644 --- a/roles/taginfo.rb +++ b/roles/taginfo.rb @@ -26,6 +26,9 @@ default_attributes( :passenger => { :max_pool_size => 50 }, + :ruby => { + :version => "3.3" + }, :taginfo => { :sites => [ { diff --git a/roles/tile.rb b/roles/tile.rb index 20bded97f..550c99f66 100644 --- a/roles/tile.rb +++ b/roles/tile.rb @@ -46,6 +46,9 @@ default_attributes( } } }, + :ruby => { + :fullstaq => false + }, :ssl => { :ct_report_uri => false }, diff --git a/roles/ucl-public.rb b/roles/ucl-public.rb new file mode 100644 index 000000000..7b7c6188a --- /dev/null +++ b/roles/ucl-public.rb @@ -0,0 +1,21 @@ +name "ucl-public" +description "Role applied to all public servers at UCL" + +default_attributes( + :networking => { + :interfaces => { + :external => { + :role => :external, + :metric => 100, + :inet => { + :prefix => "24", + :gateway => "193.60.236.254" + } + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/ucl.rb b/roles/ucl.rb index 516c925d3..1734989bf 100644 --- a/roles/ucl.rb +++ b/roles/ucl.rb @@ -4,8 +4,10 @@ description "Role applied to all servers at UCL" default_attributes( :location => "Slough, England", :networking => { - :roles => { + :interfaces => { :internal => { + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.0.3", @@ -13,13 +15,6 @@ default_attributes( "10.0.0.0/8" => { :via => "10.0.0.3" } } } - }, - :external => { - :zone => "ucl", - :inet => { - :prefix => "24", - :gateway => "193.60.236.254" - } } }, :wireguard => { diff --git a/roles/vhagar.rb b/roles/vhagar.rb index b5776ab2a..c8cd4a92f 100644 --- a/roles/vhagar.rb +++ b/roles/vhagar.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.5" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.133", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.133" }, :inet6 => { - :address => "2001:470:1:fa1::5", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::5" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.101", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.101" }, :inet6 => { - :address => "2001:4d78:500:5e3::5", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::5" } } } @@ -60,6 +40,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[nominatim]" ) diff --git a/roles/web-frontend.rb b/roles/web-frontend.rb index 396cf36db..e7c0e5850 100644 --- a/roles/web-frontend.rb +++ b/roles/web-frontend.rb @@ -30,6 +30,9 @@ default_attributes( :passenger => { :max_pool_size => 50 }, + :ruby => { + :fullstaq => true + }, :exim => { :local_domains => ["messages.openstreetmap.org"], :routes => { diff --git a/roles/wiki.rb b/roles/wiki.rb index 0c83be4c5..652726346 100644 --- a/roles/wiki.rb +++ b/roles/wiki.rb @@ -17,7 +17,7 @@ default_attributes( :max_connections_per_child => 10000 }, :evasive => { - :page_count => 250, + :page_count => 400, :site_count => 500 } }, @@ -59,7 +59,7 @@ default_attributes( :mysqld => { :innodb_buffer_pool_size => "4G", :key_buffer_size => "64M", - :max_connections => "500", + :max_connections => "800", :sort_buffer_size => "8M", :tmp_table_size => "128M" } diff --git a/roles/ysera.rb b/roles/ysera.rb index bf256d44c..8d3c16f87 100644 --- a/roles/ysera.rb +++ b/roles/ysera.rb @@ -6,14 +6,12 @@ default_attributes( :interfaces => { :internal => { :interface => "eno1np0.2801", - :role => :internal, :inet => { :address => "10.0.0.15" } }, :external => { :interface => "eno1np0.2800", - :role => :external, :inet => { :address => "193.60.236.22" } @@ -115,6 +113,6 @@ default_attributes( ) run_list( - "role[ucl]", + "role[ucl-public]", "role[tile]" ) diff --git a/test/data_bags/accounts/serverinfo.json b/test/data_bags/accounts/serverinfo.json new file mode 100644 index 000000000..2b01af603 --- /dev/null +++ b/test/data_bags/accounts/serverinfo.json @@ -0,0 +1,6 @@ +{ + "id": "serverinfo", + "uid": "534", + "comment": "hardware.openstreetmap.org", + "manage_home": false +}