From: Tom Hughes Date: Thu, 27 Mar 2025 11:35:57 +0000 (+0000) Subject: Merge remote-tracking branch 'github/pull/751' X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/HEAD?hp=8ed64eefeecb52540103054896bbff57370d1151 Merge remote-tracking branch 'github/pull/751' --- diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index e1c8df555..143ed1216 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -135,6 +135,8 @@ jobs: suite: git-server - os: debian-12 suite: git-web + - os: debian-12 + suite: gps-tile - os: debian-12 suite: imagery-tiler - os: debian-12 @@ -153,6 +155,8 @@ jobs: suite: web-frontend - os: debian-12 suite: web-rails + - os: debian-12 + suite: wiki exclude: - suite: apt-repository os: ubuntu-2204 @@ -164,6 +168,8 @@ jobs: os: ubuntu-2204 - suite: git-web os: ubuntu-2204 + - suite: gps-tile + os: ubuntu-2204 - suite: mailman os: ubuntu-2204 - suite: letsencrypt diff --git a/.kitchen.yml b/.kitchen.yml index 88cfc7459..6f4eb6bb6 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -93,6 +93,9 @@ suites: - name: blogs run_list: - recipe[blogs::default] + attributes: + ruby: + version: 3.3 - name: chef run_list: - recipe[chef::default] @@ -402,6 +405,8 @@ suites: run_list: - recipe[taginfo::default] attributes: + ruby: + version: 3.3 taginfo: sites: - name: taginfo.example.com diff --git a/Gemfile.lock b/Gemfile.lock index 40032233d..49f039957 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,20 +1,23 @@ GEM remote: https://rubygems.org/ specs: - activesupport (7.1.4.2) + activesupport (7.1.5.1) base64 + benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) mutex_m + securerandom (>= 0.3) tzinfo (~> 2.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) - ast (2.4.2) - aws-eventstream (1.3.0) + ast (2.4.3) + aws-eventstream (1.3.2) aws-partitions (1.863.0) aws-sdk-accessanalyzer (1.44.0) aws-sdk-core (~> 3, >= 3.188.0) @@ -257,7 +260,7 @@ GEM aws-sdk-core (~> 3, >= 3.188.0) aws-sigv4 (~> 1.1) aws-sigv2 (1.2.0) - aws-sigv4 (1.10.1) + aws-sigv4 (1.11.0) aws-eventstream (~> 1, >= 1.0.2) azure_graph_rbac (0.17.2) ms_rest_azure (~> 0.12.0) @@ -271,12 +274,12 @@ GEM ms_rest_azure (~> 0.12.0) base64 (0.2.0) bcrypt_pbkdf (1.1.1) - bigdecimal (3.1.8) - bson (4.15.0) + benchmark (0.4.0) + bigdecimal (3.1.9) builder (3.3.0) - chef-config (18.5.0) + chef-config (18.6.2) addressable - chef-utils (= 18.5.0) + chef-utils (= 18.6.2) fuzzyurl mixlib-config (>= 2.2.12, < 4.0) mixlib-shellout (>= 2.0, < 4.0) @@ -284,75 +287,33 @@ GEM chef-telemetry (1.1.1) chef-config concurrent-ruby (~> 1.0) - chef-utils (18.5.0) + chef-utils (18.6.2) concurrent-ruby coderay (1.1.3) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) + concurrent-ruby (1.3.5) + connection_pool (2.5.0) cookstyle (7.32.8) rubocop (= 1.25.1) declarative (0.0.20) - diff-lcs (1.5.1) - docker-api (2.3.0) + diff-lcs (1.6.1) + docker-api (2.4.0) excon (>= 0.64.0) multi_json domain_name (0.6.20240107) drb (2.2.1) - dry-configurable (1.2.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-core (1.0.1) - concurrent-ruby (~> 1.0) - zeitwerk (~> 2.6) - dry-inflector (1.1.0) - dry-logic (1.5.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-struct (1.6.0) - dry-core (~> 1.0, < 2) - dry-types (>= 1.7, < 2) - ice_nine (~> 0.11) - zeitwerk (~> 2.6) - dry-types (1.7.2) - bigdecimal (~> 3.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0) - dry-inflector (~> 1.0) - dry-logic (~> 1.4) - zeitwerk (~> 2.6) ed25519 (1.3.0) - erubi (1.13.0) - excon (0.112.0) - faraday (1.10.4) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) + erubi (1.13.1) + excon (1.2.5) + logger + faraday (1.3.1) faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + multipart-post (>= 1.2, < 3) ruby2_keywords (>= 0.0.4) faraday-cookie_jar (0.0.7) faraday (>= 0.8.0) http-cookie (~> 1.0.0) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-follow_redirects (0.3.0) - faraday (>= 1, < 3) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) faraday-net_http (1.0.2) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - faraday_middleware (1.2.1) + faraday_middleware (1.0.0) faraday (~> 1.0) ffi (1.17.0) fuzzyurl (0.9.0) @@ -389,35 +350,26 @@ GEM gyoku (1.4.0) builder (>= 2.1.2) rexml (~> 3.0) - hashdiff (1.0.1) - hashie (5.0.0) - highline (3.1.1) - reline - http-cookie (1.0.7) + hashie (4.1.0) + http-cookie (1.0.8) domain_name (~> 0.5) - httpclient (2.8.3) - i18n (1.14.6) + httpclient (2.9.0) + mutex_m + i18n (1.14.7) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) inifile (3.0.0) - inspec (5.22.58) - faraday_middleware (>= 0.12.2, < 1.3) - inspec-core (= 5.22.58) - mongo (= 2.13.2) - progress_bar (~> 1.3.3) - rake - train (~> 3.10) - train-aws (~> 0.2) + inspec (4.24.32) + faraday_middleware (>= 0.12.2, < 1.1) + inspec-core (= 4.24.32) + train (~> 3.0) + train-aws (~> 0.1) train-habitat (~> 0.1) - train-kubernetes (~> 0.1) train-winrm (~> 0.2) - inspec-core (5.22.58) + inspec-core (4.24.32) addressable (~> 2.4) - chef-telemetry (~> 1.0, >= 1.0.8) - cookstyle - faraday (>= 1, < 3) - faraday-follow_redirects (~> 0.3) - hashie (>= 3.4, < 6.0) + chef-telemetry (~> 1.0) + faraday (>= 0.9.0, < 1.4) + hashie (>= 3.4, < 5.0) license-acceptance (>= 0.2.13, < 3.0) method_source (>= 0.8, < 2.0) mixlib-log (~> 3.0) @@ -425,40 +377,27 @@ GEM parallel (~> 1.9) parslet (>= 1.5, < 3.0) pry (~> 0.13) - rspec (>= 3.9, <= 3.12) + rspec (~> 3.10) rspec-its (~> 1.2) rubyzip (>= 1.2.2, < 3.0) semverse (~> 3.0) sslshake (~> 1.2) - thor (>= 0.20, < 1.3.0) + thor (>= 0.20, < 2.0) tomlrb (>= 1.2, < 2.1) - train-core (~> 3.10) + train-core (~> 3.0) tty-prompt (~> 0.17) tty-table (~> 0.10) - io-console (0.7.2) jmespath (1.6.2) - json (2.7.3) - jsonpath (1.1.5) - multi_json - jwt (2.9.3) + json (2.10.2) + jwt (2.10.1) base64 - k8s-ruby (0.16.0) - dry-configurable - dry-struct - dry-types - excon (~> 0.71) - hashdiff (~> 1.0.0) - jsonpath (~> 1.1) - recursive-open-struct (~> 1.1.3) - yajl-ruby (~> 1.4.0) - yaml-safe_load_stream3 kitchen-dokken (2.20.7) docker-api (>= 1.33, < 3) lockfile (~> 2.1) test-kitchen (>= 1.15, < 4) - kitchen-inspec (2.6.2) + kitchen-inspec (3.0.0) hashie (>= 3.4, <= 5.0) - inspec (>= 2.2.64, < 6.0) + inspec (>= 2.2.64, < 7.0) test-kitchen (>= 2.7, < 4) license-acceptance (2.1.13) pastel (~> 0.7) @@ -467,24 +406,24 @@ GEM tty-prompt (~> 0.20) little-plugger (1.1.4) lockfile (2.1.3) + logger (1.7.0) logging (2.4.0) little-plugger (~> 1.1) multi_json (~> 1.14) method_source (1.1.0) mini_mime (1.1.5) - minitest (5.25.1) + minitest (5.25.5) mixlib-config (3.0.27) tomlrb mixlib-install (3.12.30) mixlib-shellout mixlib-versioning thor - mixlib-log (3.0.9) - mixlib-shellout (3.3.3) + mixlib-log (3.2.0) + ffi (~> 1.9, <= 1.17.0) + mixlib-shellout (3.3.8) chef-utils mixlib-versioning (1.2.12) - mongo (2.13.2) - bson (>= 4.8.2, < 5.0.0) ms_rest (0.7.6) concurrent-ruby (~> 1.0) faraday (>= 0.9, < 2.0.0) @@ -496,59 +435,52 @@ GEM ms_rest (~> 0.7.6) multi_json (1.15.0) multipart-post (2.4.1) - mutex_m (0.2.0) - net-scp (4.0.0) + mutex_m (0.3.0) + net-scp (4.1.0) net-ssh (>= 2.6.5, < 8.0.0) net-ssh (7.3.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) nori (2.7.1) bigdecimal - options (2.3.2) os (1.1.4) parallel (1.26.3) - parser (3.3.5.0) + parser (3.3.7.3) ast (~> 2.4.1) racc parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - progress_bar (1.3.4) - highline (>= 1.6) - options (~> 2.3.0) - pry (0.14.2) + prism (1.4.0) + pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) public_suffix (6.0.1) racc (1.8.1) rainbow (3.1.1) - rake (13.2.1) - recursive-open-struct (1.1.3) - regexp_parser (2.9.2) - reline (0.5.10) - io-console (~> 0.5) + regexp_parser (2.10.0) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.3.9) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.3) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.4) + rexml (3.4.1) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.3) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.3) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) + rspec-support (~> 3.13.0) rspec-its (1.3.1) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.12.7) + rspec-mocks (3.13.2) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.2) + rspec-support (~> 3.13.0) + rspec-support (3.13.2) rubocop (1.25.1) parallel (~> 1.10) parser (>= 3.1.0.0) @@ -558,13 +490,15 @@ GEM rubocop-ast (>= 1.15.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.32.3) - parser (>= 3.3.1.0) + rubocop-ast (1.43.0) + parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyntlm (0.6.5) base64 - rubyzip (2.3.2) + rubyzip (2.4.1) + securerandom (0.4.1) semverse (3.0.2) signet (0.19.0) addressable (~> 2.8) @@ -591,7 +525,7 @@ GEM winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.1) - thor (1.2.2) + thor (1.3.2) timeliness (0.3.10) tomlrb (1.3.0) trailblazer-option (0.1.2) @@ -703,9 +637,6 @@ GEM net-scp (>= 1.2, < 5.0) net-ssh (>= 2.9, < 8.0) train-habitat (0.2.22) - train-kubernetes (0.2.1) - k8s-ruby (~> 0.16.0) - train (~> 3.0) train-winrm (0.2.13) winrm (>= 2.3.6, < 3.0) winrm-elevated (~> 1.2.2) @@ -753,8 +684,6 @@ GEM rubyzip (~> 2.0) winrm (~> 2.0) wisper (2.0.1) - yajl-ruby (1.4.3) - yaml-safe_load_stream3 (0.1.2) zeitwerk (2.6.18) PLATFORMS diff --git a/cookbooks/apt/recipes/fullstaq-ruby.rb b/cookbooks/apt/recipes/fullstaq-ruby.rb new file mode 100644 index 000000000..d9dad7d37 --- /dev/null +++ b/cookbooks/apt/recipes/fullstaq-ruby.rb @@ -0,0 +1,27 @@ +# +# Cookbook:: apt +# Recipe:: fullstaq-ruby +# +# Copyright:: 2025, Tom Hughes +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "apt" + +apt_repository "fullstaq-ruby" do + uri "https://apt.fullstaqruby.org" + distribution "#{node[:platform]}-#{node[:platform_version]}" + components ["main"] + key "https://raw.githubusercontent.com/fullstaq-ruby/server-edition/main/fullstaq-ruby.asc" +end diff --git a/cookbooks/bind/templates/default/db.10.erb b/cookbooks/bind/templates/default/db.10.erb index 06aef6442..298811c68 100644 --- a/cookbooks/bind/templates/default/db.10.erb +++ b/cookbooks/bind/templates/default/db.10.erb @@ -2,7 +2,7 @@ $TTL 604800 @ IN SOA <%= node[:fdqn] %>. root.openstreetmap.org. ( - 2021092001 ; Serial + 2025021801 ; Serial 604800 ; Refresh 86400 ; Retry 2419200 ; Expire @@ -44,6 +44,7 @@ $TTL 604800 100.48.0 IN PTR pdu1.ams.openstreetmap.org. 101.48.0 IN PTR pdu2.ams.openstreetmap.org. 102.48.0 IN PTR oob1.ams.openstreetmap.org. +103.48.0 IN PTR ats1.ams.openstreetmap.org. 3.49.0 IN PTR faffy.oob.openstreetmap.org. 4.49.0 IN PTR dribble.oob.openstreetmap.org. diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index 0fb2cc7aa..5c650f242 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -44,10 +44,18 @@ git "/srv/blogs.openstreetmap.org" do group "blogs" end +bundle_config "/srv/blogs.openstreetmap.org" do + action :nothing + user "blogs" + group "blogs" + settings "deployment" => "true", + "without" => "development:test", + "build.sqlite3" => "--enable-system-libraries" + subscribes :create, "git[/srv/blogs.openstreetmap.org]", :immediately +end + bundle_install "/srv/blogs.openstreetmap.org" do action :nothing - options "--deployment --without development test" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately @@ -56,7 +64,6 @@ end bundle_exec "/srv/blogs.openstreetmap.org" do action :nothing command "pluto build -t osm -o build" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately diff --git a/cookbooks/blogs/templates/default/blogs-update.erb b/cookbooks/blogs/templates/default/blogs-update.erb index a7d021495..15cc82483 100644 --- a/cookbooks/blogs/templates/default/blogs-update.erb +++ b/cookbooks/blogs/templates/default/blogs-update.erb @@ -2,8 +2,6 @@ cd /srv/blogs.openstreetmap.org -export BUNDLE_PATH="vendor/bundle" - <%= node[:ruby][:bundle] %> exec pluto \ --quieter \ --config=/srv/blogs.openstreetmap.org build \ diff --git a/cookbooks/community/recipes/default.rb b/cookbooks/community/recipes/default.rb index 51c4624ea..55238e824 100644 --- a/cookbooks/community/recipes/default.rb +++ b/cookbooks/community/recipes/default.rb @@ -63,7 +63,7 @@ git "/srv/community.openstreetmap.org/docker" do action :sync repository "https://github.com/discourse/discourse_docker.git" # DANGER launch wrapper automatically updates git repo if rebuild method used: https://github.com/discourse/discourse_docker/blob/107ffb40fe8b1ea40e00814468db974a4f3f8e8f/launcher#L799 - revision "136c63890674b95df1327d24270c55e4ef8e87a8" + revision "e42fa9711e9a8b27e9618342b5b456d3ba5b8025" user "root" group "root" notifies :run, "notify_group[discourse_container_new_data]" @@ -155,6 +155,7 @@ execute "discourse_container_data_bootstrap" do cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" + retries 2 # Postgres upgrades required a second run end execute "discourse_container_data_destroy" do diff --git a/cookbooks/community/templates/default/data.yml.erb b/cookbooks/community/templates/default/data.yml.erb index c5c59d288..37573874e 100644 --- a/cookbooks/community/templates/default/data.yml.erb +++ b/cookbooks/community/templates/default/data.yml.erb @@ -3,7 +3,7 @@ # templates: - - "templates/postgres.13.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW + - "templates/postgres.15.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW - "templates/redis.template.yml" # any extra arguments for Docker? @@ -51,6 +51,6 @@ hooks: run: # Make sure this matches the postgresql version template above - replace: - filename: "/etc/postgresql/13/main/postgresql.conf" + filename: "/etc/postgresql/15/main/postgresql.conf" from: /#?max_connections *=.*/ to: "max_connections = $db_max_connections" diff --git a/cookbooks/community/templates/default/web_only.yml.erb b/cookbooks/community/templates/default/web_only.yml.erb index 5beef4802..587507a43 100644 --- a/cookbooks/community/templates/default/web_only.yml.erb +++ b/cookbooks/community/templates/default/web_only.yml.erb @@ -19,7 +19,7 @@ links: # any extra arguments for Docker? # docker_args: -# Latest Version v3.3.3 +# Latest Version v3.4.2 # Discourse only support tests-passed and stable branches params: version: stable @@ -116,14 +116,6 @@ hooks: cmd: - sudo -H -E -u discourse cp /shared/feeds/update-feeds.atom public/update-feeds.atom after_ssl: - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /listen 80;/ - to: | - listen 80; - listen [::]:80; - rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; - - replace: filename: "/etc/nginx/conf.d/discourse.conf" from: /add_header.+/ @@ -133,3 +125,12 @@ hooks: resolver <%= @resolvers.join(" ") %>; resolver_timeout 5s; ssl_dhparam /shared/ssl/dhparam.pem; + +run: + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /listen 80;/ + to: | + listen 80; + listen [::]:80; + rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index 0d6790f46..f42c22ee3 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -203,7 +203,7 @@ template "/srv/dev.openstreetmap.org/index.html" do end ssl_certificate "dev.openstreetmap.org" do - domains "dev.openstreetmap.org" + domains ["dev.openstreetmap.org", "dev.osm.org"] notifies :reload, "service[apache2]" end @@ -225,7 +225,7 @@ file "/etc/apache2/conf.d/phppgadmin" do end ssl_certificate "phppgadmin.dev.openstreetmap.org" do - domains "phppgadmin.dev.openstreetmap.org" + domains ["phppgadmin.dev.openstreetmap.org", "phppgadmin.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -351,7 +351,7 @@ if node[:postgresql][:clusters][:"15/main"] gpx_directory = "#{site_directory}/gpx" if details[:repository] - site_aliases = details[:aliases] || [] + site_aliases = details[:aliases] || ["#{name}.apis.dev.osm.org"] secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do @@ -572,7 +572,7 @@ if node[:postgresql][:clusters][:"15/main"] end ssl_certificate "apis.dev.openstreetmap.org" do - domains "apis.dev.openstreetmap.org" + domains ["apis.dev.openstreetmap.org", "apis.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -601,7 +601,8 @@ ssl_certificate "ooc.openstreetmap.org" do domains ["ooc.openstreetmap.org", "a.ooc.openstreetmap.org", "b.ooc.openstreetmap.org", - "c.ooc.openstreetmap.org"] + "c.ooc.openstreetmap.org", + "ooc.osm.org"] notifies :reload, "service[apache2]" end diff --git a/cookbooks/dev/templates/default/apache.apis.erb b/cookbooks/dev/templates/default/apache.apis.erb index 4d78fdd6a..470a1fa5f 100644 --- a/cookbooks/dev/templates/default/apache.apis.erb +++ b/cookbooks/dev/templates/default/apache.apis.erb @@ -2,6 +2,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -16,6 +18,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/apis.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.dev.erb b/cookbooks/dev/templates/default/apache.dev.erb index 2d3d9473d..2dd13c2b9 100644 --- a/cookbooks/dev/templates/default/apache.dev.erb +++ b/cookbooks/dev/templates/default/apache.dev.erb @@ -2,6 +2,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -21,6 +23,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.ooc.erb b/cookbooks/dev/templates/default/apache.ooc.erb index 0a9b0b979..a532300e1 100644 --- a/cookbooks/dev/templates/default/apache.ooc.erb +++ b/cookbooks/dev/templates/default/apache.ooc.erb @@ -5,6 +5,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -26,6 +28,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/ooc.openstreetmap.org-access.log combined_extended @@ -37,6 +41,8 @@ ServerName npe.openstreetmap.org + ServerAlias npe.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/npe.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.phppgadmin.erb b/cookbooks/dev/templates/default/apache.phppgadmin.erb index 546a05cbe..67c37df16 100644 --- a/cookbooks/dev/templates/default/apache.phppgadmin.erb +++ b/cookbooks/dev/templates/default/apache.phppgadmin.erb @@ -2,6 +2,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -23,6 +25,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/phppgadmin.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.rails.erb b/cookbooks/dev/templates/default/apache.rails.erb index 957866c5c..082931dde 100644 --- a/cookbooks/dev/templates/default/apache.rails.erb +++ b/cookbooks/dev/templates/default/apache.rails.erb @@ -48,6 +48,7 @@ RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] <% end -%> diff --git a/cookbooks/dev/templates/default/apache.user.erb b/cookbooks/dev/templates/default/apache.user.erb index 373d12258..0c51e44e0 100644 --- a/cookbooks/dev/templates/default/apache.user.erb +++ b/cookbooks/dev/templates/default/apache.user.erb @@ -4,9 +4,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + SSLEngine on SSLCertificateFile /etc/ssl/certs/<%= @user %>.dev.openstreetmap.org.pem SSLCertificateKeyFile /etc/ssl/private/<%= @user %>.dev.openstreetmap.org.key @@ -36,9 +37,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + CustomLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-access.log combined_extended ErrorLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-error.log diff --git a/cookbooks/dhcpd/recipes/default.rb b/cookbooks/dhcpd/recipes/default.rb index 0e6b9ec3f..685d08c3b 100644 --- a/cookbooks/dhcpd/recipes/default.rb +++ b/cookbooks/dhcpd/recipes/default.rb @@ -53,7 +53,7 @@ remote_file "/srv/tftp/netboot.xyz.kpxe" do mode "644" end -domain = "#{node[:networking][:roles][:external][:zone]}.openstreetmap.org" +domain = node[:networking][:search].first template "/etc/dhcp/dhcpd.conf" do source "dhcpd.conf.erb" diff --git a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb index 1581475e5..ed4fe2d03 100644 --- a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb +++ b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb @@ -55,6 +55,12 @@ host pdu2.ams.openstreetmap.org { fixed-address 10.0.48.101; } +host ats1.ams.openstreetmap.org { + hardware ethernet 00:c0:b7:e5:5e:f1; + server-name "ats1.ams.openstreetmap.org"; + fixed-address 10.0.48.103; +} + host oob1.dub.openstreetmap.org { hardware ethernet 62:bd:62:a6:05:25; server-name "oob1.dub.openstreetmap.org"; diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index 3a19d8a91..0ec59c8b3 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -42,7 +42,7 @@ package %w[ cache_dir = Chef::Config[:file_cache_path] -dnscontrol_version = "4.15.5" +dnscontrol_version = "4.17.0" dnscontrol_arch = if arm? "arm64" diff --git a/cookbooks/gps-tile/templates/default/apache.erb b/cookbooks/gps-tile/templates/default/apache.erb index 9adf707dd..a1b7ba94c 100644 --- a/cookbooks/gps-tile/templates/default/apache.erb +++ b/cookbooks/gps-tile/templates/default/apache.erb @@ -47,6 +47,7 @@ RewriteRule ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 [R=permanent,L] # Redirect to https + RewriteCond %{REQUEST_URI} !^/server-status RewriteRule (.*) https://%{SERVER_NAME}/$1 [R=permanent,L] # Setup logging diff --git a/cookbooks/imagery/recipes/au_act_aerial.rb b/cookbooks/imagery/recipes/au_act_aerial.rb new file mode 100644 index 000000000..cc0135998 --- /dev/null +++ b/cookbooks/imagery/recipes/au_act_aerial.rb @@ -0,0 +1,65 @@ +# +# Cookbook:: imagery +# Recipe:: au_act_aerial +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "act-imagery.openstreetmap.org" do + title "OpenStreetMap - ACT Imagery" + aliases ["act-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-35.942, 148.729], [-35.117, 149.430]] +end + +imagery_layer "act_aerial_imagery_202409" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202409" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + default_layer true + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202311" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202311" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202305" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202305" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end diff --git a/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb new file mode 100644 index 000000000..cda5e80d4 --- /dev/null +++ b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb @@ -0,0 +1,59 @@ +# +# Cookbook:: imagery +# Recipe:: au_vic_melbourne_aerial +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "au-vic-melbourne-imagery.openstreetmap.org" do + title "OpenStreetMap - City of Melbourne - Aerial Imagery" + aliases ["au-vic-melbourne-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-37.850667, 144.896981], [-37.775451, 144.991351]] +end + +imagery_layer "melbourne-2020" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2020" + source "/store/imagery/au/city-of-melbourne/CoM_May2020_2cm.cog.tiff" + copyright "(c) 2020 City of Melbourne" + max_zoom 23 + extension "jpeg" + revision 2 + default_layer true +end + +imagery_layer "melbourne-2019" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2019" + source "/store/imagery/au/city-of-melbourne/CoM_03Feb2019.cog.tiff" + copyright "(c) 2019 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end + +imagery_layer "melbourne-2018" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2018" + source "/store/imagery/au/city-of-melbourne/CoM_May2018_10cm.COG.tiff" + copyright "(c) 2018 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end diff --git a/cookbooks/imagery/recipes/tiler.rb b/cookbooks/imagery/recipes/tiler.rb index 319cb5794..571425bda 100644 --- a/cookbooks/imagery/recipes/tiler.rb +++ b/cookbooks/imagery/recipes/tiler.rb @@ -63,7 +63,7 @@ end systemd_timer "titiler-restart" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "2h" randomized_delay_sec "20m" end diff --git a/cookbooks/imagery/recipes/za_ngi_topo.rb b/cookbooks/imagery/recipes/za_ngi_topo.rb index a1fd92c61..ac5a28b37 100644 --- a/cookbooks/imagery/recipes/za_ngi_topo.rb +++ b/cookbooks/imagery/recipes/za_ngi_topo.rb @@ -32,6 +32,7 @@ imagery_layer "za_ngi_topo_250k" do source "/store/imagery/za/ngi-topo-250k/ngi-topo-250k-combined.vrt" copyright 'State Copyright © 1996–2010 Chief Directorate: National Geo-spatial Information' default_layer true + revision 2 end imagery_layer "za_ngi_topo_50k" do @@ -40,4 +41,5 @@ imagery_layer "za_ngi_topo_50k" do projection "EPSG:3857" source "/store/imagery/za/ngi-topo-50k/ngi-topo-50k-combined.vrt" copyright 'State Copyright © 1996–2013 Chief Directorate: National Geo-spatial Information' + revision 2 end diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index 5eddb448c..7151c68c6 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -105,15 +105,19 @@ action :create do description "Map server for #{new_resource.site} layer" environment "MS_DEBUGLEVEL" => "0", "MS_ERRORFILE" => "stderr", - "GDAL_CACHEMAX" => "128" + "GDAL_CACHEMAX" => "128", + "GDAL_HTTP_TCP_KEEPALIVE" => "YES", + "GDAL_HTTP_VERSION" => "2TLS", + "GDAL_ENABLE_WMS_CACHE" => "NO" limit_nofile 16384 memory_high "12G" memory_max "12G" + limit_core 0 user "imagery" group "imagery" exec_start "/usr/bin/multiwatch -f 8 --signal=TERM -- /usr/lib/cgi-bin/mapserv" standard_input "socket" - sandbox true + sandbox :enable_network => true restrict_address_families "AF_UNIX" timeout_stop_sec 60 not_if { new_resource.uses_tiler } @@ -154,7 +158,7 @@ action :create do systemd_timer "mapserv-fcgi-#{new_resource.site}-stop" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "6h" randomized_delay_sec "20m" not_if { new_resource.uses_tiler } end diff --git a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb index 03a439343..a232d7830 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb @@ -23,8 +23,11 @@ upstream <%= @name %>_fastcgi { <% end -%> server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; + server_name <%= @name %> a.<%= @name %> b.<%= @name %> c.<%= @name %><% @aliases.each do |alias_name| %> <%= alias_name %> a.<%= alias_name %> b.<%= alias_name %> c.<%= alias_name %><%- end -%>; http2_max_concurrent_streams 512; diff --git a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb index 723d0d0f2..adc2d23c0 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb @@ -3,7 +3,7 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" { <% if @uses_tiler -%> set $args ""; - rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp) /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=$4 break; + rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp) /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x.$4?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first break; proxy_pass http://<%= @site %>_tiler_backend; proxy_set_header Host $host; proxy_set_header Referer $http_referer; @@ -66,8 +66,8 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" { gzip off; # Add HTTP Cache-Control + Expires Headers - expires 7d; - add_header Cache-Control "stale-while-revalidate=604800, stale-if-error=604800"; + expires 31d; + add_header Cache-Control "stale-while-revalidate=2678400, stale-if-error=2678400"; add_header x-cache-status $upstream_cache_status; # Allow CORS requests diff --git a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb index 155620949..055c2f849 100644 --- a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb @@ -21,8 +21,11 @@ upstream titiler_api_backend { } server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; + server_name <%= @name %> <% @aliases.each do |alias_name| %> <%= alias_name %><%- end -%>; http2_max_concurrent_streams 512; diff --git a/cookbooks/letsencrypt/files/default/bin/renew-hook b/cookbooks/letsencrypt/files/default/bin/deploy-hook similarity index 100% rename from cookbooks/letsencrypt/files/default/bin/renew-hook rename to cookbooks/letsencrypt/files/default/bin/deploy-hook diff --git a/cookbooks/letsencrypt/files/default/bin/renew b/cookbooks/letsencrypt/files/default/bin/renew index 6a0482185..f3c7cd0cd 100755 --- a/cookbooks/letsencrypt/files/default/bin/renew +++ b/cookbooks/letsencrypt/files/default/bin/renew @@ -1,10 +1,9 @@ #!/bin/sh -cd /srv/acme.openstreetmap.org - /usr/bin/certbot renew \ + --key-type ecdsa \ --quiet \ --config-dir /srv/acme.openstreetmap.org/config \ --work-dir /srv/acme.openstreetmap.org/work \ --logs-dir /srv/acme.openstreetmap.org/logs \ - --renew-hook /srv/acme.openstreetmap.org/bin/renew-hook + --deploy-hook /srv/acme.openstreetmap.org/bin/deploy-hook diff --git a/cookbooks/letsencrypt/metadata.rb b/cookbooks/letsencrypt/metadata.rb index 52ef99f7c..a826ce5e4 100644 --- a/cookbooks/letsencrypt/metadata.rb +++ b/cookbooks/letsencrypt/metadata.rb @@ -9,3 +9,4 @@ supports "ubuntu" depends "accounts" depends "apache" depends "chef" +depends "ruby" diff --git a/cookbooks/letsencrypt/recipes/default.rb b/cookbooks/letsencrypt/recipes/default.rb index 382a0a58c..3e492f6af 100644 --- a/cookbooks/letsencrypt/recipes/default.rb +++ b/cookbooks/letsencrypt/recipes/default.rb @@ -20,13 +20,11 @@ include_recipe "accounts" include_recipe "apache" include_recipe "chef::knife" +include_recipe "ruby" keys = data_bag_item("chef", "keys") -package %w[ - certbot - ruby -] +package "certbot" directory "/etc/letsencrypt" do owner "letsencrypt" @@ -116,6 +114,13 @@ remote_directory "/srv/acme.openstreetmap.org/bin" do files_mode "755" end +template "/srv/acme.openstreetmap.org/bin/upload" do + source "upload.erb" + owner "root" + group "root" + mode "755" +end + directory "/srv/acme.openstreetmap.org/requests" do owner "root" group "root" @@ -168,6 +173,13 @@ Dir.glob("*", :base => "/srv/acme.openstreetmap.org/requests") do |name| end end +template "/srv/acme.openstreetmap.org/bin/check-certificate" do + source "check-certificate.erb" + owner "root" + group "root" + mode "755" +end + template "/srv/acme.openstreetmap.org/bin/check-certificates" do source "check-certificates.erb" owner "root" diff --git a/cookbooks/letsencrypt/files/default/bin/check-certificate b/cookbooks/letsencrypt/templates/default/check-certificate.erb old mode 100755 new mode 100644 similarity index 93% rename from cookbooks/letsencrypt/files/default/bin/check-certificate rename to cookbooks/letsencrypt/templates/default/check-certificate.erb index f24681589..319072b6b --- a/cookbooks/letsencrypt/files/default/bin/check-certificate +++ b/cookbooks/letsencrypt/templates/default/check-certificate.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "socket" require "openssl" @@ -33,6 +33,10 @@ if ssl puts "Certificate #{domains.first} on #{host} expires at #{certificate.not_after}" end + unless certificate.public_key.is_a?(OpenSSL::PKey::EC) + puts "Certificate #{domains.first} on #{host} does not use ECDSA key type" + end + digest = OpenSSL::Digest::SHA1.new certificate_id = OpenSSL::OCSP::CertificateId.new(certificate, issuer, digest) ocsp_request = OpenSSL::OCSP::Request.new.add_certid(certificate_id) diff --git a/cookbooks/letsencrypt/templates/default/request.erb b/cookbooks/letsencrypt/templates/default/request.erb index eaefa5bbe..65e80b303 100644 --- a/cookbooks/letsencrypt/templates/default/request.erb +++ b/cookbooks/letsencrypt/templates/default/request.erb @@ -2,6 +2,11 @@ # DO NOT EDIT - This file is being maintained by Chef +if [ "$(id -un)" != "letsencrypt" ]; then + echo "Error: This script must be run as user letsencrypt" >&2 + exit 1 +fi + /usr/bin/certbot certonly \ --non-interactive \ --config-dir /srv/acme.openstreetmap.org/config \ @@ -10,12 +15,12 @@ --email operations@osmfoundation.org \ --agree-tos \ --expand \ + --renew-with-new-domains \ + --cert-name <%= @domains.first %> \ <% @domains.each do |domain| -%> --domain <%= domain %> \ <% end -%> --webroot \ - --webroot-path /srv/acme.openstreetmap.org/html - -/srv/acme.openstreetmap.org/bin/upload \ - <%= @domains.first %> \ - /srv/acme.openstreetmap.org/config/live/<%= @domains.first %> + --webroot-path /srv/acme.openstreetmap.org/html \ + --deploy-hook /srv/acme.openstreetmap.org/bin/deploy-hook \ + "$@" diff --git a/cookbooks/letsencrypt/files/default/bin/upload b/cookbooks/letsencrypt/templates/default/upload.erb old mode 100755 new mode 100644 similarity index 67% rename from cookbooks/letsencrypt/files/default/bin/upload rename to cookbooks/letsencrypt/templates/default/upload.erb index a52f30257..e89912c4f --- a/cookbooks/letsencrypt/files/default/bin/upload +++ b/cookbooks/letsencrypt/templates/default/upload.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "json" require "tempfile" @@ -17,4 +17,4 @@ file = Tempfile.new(["letsencrypt", ".json"]) file.puts JSON.generate(bag) file.close -system("/opt/chef/embedded/bin/knife", "data", "bag", "from", "file", "letsencrypt", file.path) +system("/opt/chef/embedded/bin/knife", "data", "bag", "from", "file", "letsencrypt", file.path, "--config", "/srv/acme.openstreetmap.org/.chef/knife.rb", "--key", "/srv/acme.openstreetmap.org/.chef/client.pem") diff --git a/cookbooks/mediawiki/resources/extension.rb b/cookbooks/mediawiki/resources/extension.rb index 6240f6e87..fea8614d2 100644 --- a/cookbooks/mediawiki/resources/extension.rb +++ b/cookbooks/mediawiki/resources/extension.rb @@ -74,7 +74,7 @@ action :create do user node[:mediawiki][:user] group node[:mediawiki][:group] mode "664" - variables new_resource.variables + variables new_resource.variables.merge(:site => new_resource.site) end else file "#{mediawiki_directory}/LocalSettings.d/Ext-#{new_resource.extension}.inc.php" do diff --git a/cookbooks/mediawiki/resources/site.rb b/cookbooks/mediawiki/resources/site.rb index 096484a99..840016fa4 100644 --- a/cookbooks/mediawiki/resources/site.rb +++ b/cookbooks/mediawiki/resources/site.rb @@ -44,6 +44,9 @@ property :private_site, :kind_of => [TrueClass, FalseClass], :default => false property :hcaptcha_public_key, :kind_of => String, :default => "" property :hcaptcha_private_key, :kind_of => String, :default => "" property :extra_file_extensions, :kind_of => [String, Array], :default => [] +property :namespaces, :kind_of => Hash, :default => {} +property :force_ui_messages, :kind_of => Array, :default => [] +property :watch_category_membership, :kind_of => [TrueClass, FalseClass], :default => false property :fpm_max_children, :kind_of => Integer, :default => 5 property :fpm_start_servers, :kind_of => Integer, :default => 2 property :fpm_min_spare_servers, :kind_of => Integer, :default => 1 @@ -96,6 +99,7 @@ action :create do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "664" + variables :version => new_resource.version end execute "#{mediawiki_directory}/composer.json" do @@ -559,7 +563,8 @@ action_class do end def mediawiki_reference - shell_out!("git", "ls-remote", "--refs", "--sort=-version:refname", + shell_out!("git", "-c", "versionsort.suffix=-rc", + "ls-remote", "--refs", "--sort=-version:refname", "https://gerrit.wikimedia.org/r/mediawiki/core.git", "refs/tags/#{new_resource.version}.*") .stdout @@ -596,7 +601,10 @@ action_class do :site_readonly => new_resource.site_readonly, :extra_file_extensions => new_resource.extra_file_extensions, :private_accounts => new_resource.private_accounts, - :private_site => new_resource.private_site + :private_site => new_resource.private_site, + :namespaces => new_resource.namespaces, + :force_ui_messages => new_resource.force_ui_messages, + :watch_category_membership => new_resource.watch_category_membership } end diff --git a/cookbooks/mediawiki/templates/default/LocalSettings.php.erb b/cookbooks/mediawiki/templates/default/LocalSettings.php.erb index 93aab016d..d70119627 100644 --- a/cookbooks/mediawiki/templates/default/LocalSettings.php.erb +++ b/cookbooks/mediawiki/templates/default/LocalSettings.php.erb @@ -296,92 +296,22 @@ $wgSiteNotice = "<%= @mediawiki[:site_notice] %>"; $wgReadOnly = "<%= @mediawiki[:site_readonly] %>"; <% end -%> -<% if @name == "wiki.openstreetmap.org" -%> -# DE -define('NS_LANG_DE', 200); -$wgExtraNamespaces[NS_LANG_DE] = 'DE'; -$wgNamespacesWithSubpages[NS_LANG_DE] = TRUE; -$wgContentNamespaces[] = NS_LANG_DE; -define('NS_LANG_DE_TALK', 201); -$wgExtraNamespaces[NS_LANG_DE_TALK] = 'DE_talk'; -$wgNamespacesWithSubpages[NS_LANG_DE_TALK] = TRUE; - -# FR -define('NS_LANG_FR', 202); -$wgExtraNamespaces[NS_LANG_FR] = 'FR'; -$wgNamespacesWithSubpages[NS_LANG_FR] = TRUE; -$wgContentNamespaces[] = NS_LANG_FR; -define('NS_LANG_FR_TALK', 203); -$wgExtraNamespaces[NS_LANG_FR_TALK] = 'FR_talk'; -$wgNamespacesWithSubpages[NS_LANG_FR_TALK] = TRUE; - -# ES -define('NS_LANG_ES', 204); -$wgExtraNamespaces[NS_LANG_ES] = 'ES'; -$wgNamespacesWithSubpages[NS_LANG_ES] = TRUE; -$wgContentNamespaces[] = NS_LANG_ES; -define('NS_LANG_ES_TALK', 205); -$wgExtraNamespaces[NS_LANG_ES_TALK] = 'ES_talk'; -$wgNamespacesWithSubpages[NS_LANG_ES_TALK] = TRUE; - -# IT -define('NS_LANG_IT', 206); -$wgExtraNamespaces[NS_LANG_IT] = 'IT'; -$wgNamespacesWithSubpages[NS_LANG_IT] = TRUE; -$wgContentNamespaces[] = NS_LANG_IT; -define('NS_LANG_IT_TALK', 207); -$wgExtraNamespaces[NS_LANG_IT_TALK] = 'IT_talk'; -$wgNamespacesWithSubpages[NS_LANG_IT_TALK] = TRUE; - -# NL -define('NS_LANG_NL', 208); -$wgExtraNamespaces[NS_LANG_NL] = 'NL'; -$wgNamespacesWithSubpages[NS_LANG_NL] = TRUE; -$wgContentNamespaces[] = NS_LANG_NL; -define('NS_LANG_NL_TALK', 209); -$wgExtraNamespaces[NS_LANG_NL_TALK] = 'NL_talk'; -$wgNamespacesWithSubpages[NS_LANG_NL_TALK] = TRUE; - -# RU -define('NS_LANG_RU', 210); -$wgExtraNamespaces[NS_LANG_RU] = 'RU'; -$wgNamespacesWithSubpages[NS_LANG_RU] = TRUE; -$wgContentNamespaces[] = NS_LANG_RU; -define('NS_LANG_RU_TALK', 211); -$wgExtraNamespaces[NS_LANG_RU_TALK] = 'RU_talk'; -$wgNamespacesWithSubpages[NS_LANG_RU_TALK] = TRUE; - -# JA -define('NS_LANG_JA', 212); -$wgExtraNamespaces[NS_LANG_JA] = 'JA'; -$wgNamespacesWithSubpages[NS_LANG_JA] = TRUE; -$wgContentNamespaces[] = NS_LANG_JA; -define('NS_LANG_JA_TALK', 213); -$wgExtraNamespaces[NS_LANG_JA_TALK] = 'JA_talk'; -$wgNamespacesWithSubpages[NS_LANG_JA_TALK] = TRUE; - -# Proposal -# namespace features a specific search weight defined at -# cookbooks/mediawiki/templates/default/mw-ext-CirrusSearch.inc.php.erb -define('NS_PROPOSAL', 3000); -$wgExtraNamespaces[NS_PROPOSAL] = 'Proposal'; -$wgNamespacesWithSubpages[NS_PROPOSAL] = TRUE; -$wgContentNamespaces[] = NS_PROPOSAL; -define('NS_PROPOSAL_TALK', 3001); -$wgExtraNamespaces[NS_PROPOSAL_TALK] = 'Proposal_talk'; -$wgNamespacesWithSubpages[NS_PROPOSAL_TALK] = TRUE; - -$wgNamespacesToBeSearchedDefault[NS_LANG_DE] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_FR] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_ES] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_IT] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_NL] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_RU] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_JA] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_PROPOSAL] = TRUE; +<% @mediawiki[:namespaces].each do |name, details| -%> +# <%= name %> +define('NS_<%= name.upcase %>', <%= details[:id] %>); +$wgExtraNamespaces[NS_<%= name.upcase %>] = '<%= name %>'; +$wgNamespacesWithSubpages[NS_<%= name.upcase %>] = TRUE; +$wgContentNamespaces[] = NS_<%= name.upcase %>; +$wgNamespacesToBeSearchedDefault[NS_<%= name.upcase %>] = TRUE; +define('NS_<%= name.upcase %>_TALK', <%= details[:talk_id] %>); +$wgExtraNamespaces[NS_<%= name.upcase %>_TALK] = '<%= name %>_talk'; +$wgNamespacesWithSubpages[NS_<%= name.upcase %>_TALK] = TRUE; +<% end -%> + +<% unless @mediawiki[:force_ui_messages].empty? -%> # defines which links of the sidebar are translatable -$wgForceUIMsgAsContentMsg = array( 'mainpage-url', 'mapfeatures-url', 'contributors-url', 'helppage', 'blogs-url', 'shop-url', 'sitesupport-url' ); +$wgForceUIMsgAsContentMsg = array( '<%= @mediawiki[:force_ui_messages].join("', '") %>' ); <% end -%> # load extensions @@ -389,8 +319,8 @@ $wgForceUIMsgAsContentMsg = array( 'mainpage-url', 'mapfeatures-url', 'contribut <%= "require_once('#{file}');" %> <% end -%> -<% if @name == "wiki.openstreetmap.org" -%> -# wiki.openstreetmap.org specific config loaded after extensions +<% if @mediawiki[:watch_category_membership] -%> +# must be after extensions are loaded $wgRCWatchCategoryMembership = true; <% end -%> diff --git a/cookbooks/mediawiki/templates/default/composer.local.json.erb b/cookbooks/mediawiki/templates/default/composer.local.json.erb index 73678e1cc..37382c5f6 100644 --- a/cookbooks/mediawiki/templates/default/composer.local.json.erb +++ b/cookbooks/mediawiki/templates/default/composer.local.json.erb @@ -8,6 +8,15 @@ } }, "require": { - "guzzlehttp/psr7": "2.4.5" +<% if @version.to_f < 1.43 -%> + "guzzlehttp/psr7": "2.4.5", + "data-values/common": "1.0.0", + "data-values/data-values": "3.0.0", + "data-values/geo": "4.2.3", + "data-values/interfaces": "1.0.0", + "data-values/number": "0.11.1", + "data-values/serialization": "1.2.4", + "data-values/time": "1.0.4" +<% end -%> } } diff --git a/cookbooks/networking/attributes/default.rb b/cookbooks/networking/attributes/default.rb index 9832ce8f3..06511d462 100644 --- a/cookbooks/networking/attributes/default.rb +++ b/cookbooks/networking/attributes/default.rb @@ -6,11 +6,10 @@ default[:networking][:firewall][:outgoing] = [] default[:networking][:firewall][:http_rate_limit] = nil default[:networking][:firewall][:http_connection_limit] = nil default[:networking][:firewall][:allowlist] = [] -default[:networking][:roles] = {} default[:networking][:interfaces] = {} default[:networking][:nameservers] = %w[8.8.8.8 8.8.4.4 2001:4860:4860::8888 2001:4860:4860::8844] default[:networking][:search] = [] -default[:networking][:dnssec] = "allow-downgrade" +default[:networking][:dnssec] = "false" default[:networking][:hostname] = node.name default[:networking][:wireguard][:enabled] = true default[:networking][:wireguard][:keepalive] = 180 diff --git a/cookbooks/networking/recipes/default.rb b/cookbooks/networking/recipes/default.rb index 2b4d96761..64727d091 100644 --- a/cookbooks/networking/recipes/default.rb +++ b/cookbooks/networking/recipes/default.rb @@ -23,8 +23,6 @@ require "ipaddr" require "yaml" -include_recipe "ruby" - keys = data_bag_item("networking", "keys") file "/etc/netplan/00-installer-config.yaml" do @@ -59,37 +57,18 @@ interfaces = node[:networking][:interfaces].collect do |name, interface| [interface[:interface], name] end.to_h -node[:networking][:interfaces].each do |name, interface| - if interface[:interface] =~ /^(.*)\.(\d+)$/ - vlan_interface = Regexp.last_match(1) - vlan_id = Regexp.last_match(2) - - parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" - - node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface - node.default_unless[:networking][:interfaces][parent][:vlans] = [] - - node.default[:networking][:interfaces][parent][:vlans] << vlan_id - end +node[:networking][:interfaces].each do |_, interface| + next unless interface[:interface] =~ /^(.*)\.(\d+)$/ - next unless interface[:role] && (role = node[:networking][:roles][interface[:role]]) + vlan_interface = Regexp.last_match(1) + vlan_id = Regexp.last_match(2) - if interface[:inet] && role[:inet] - node.default_unless[:networking][:interfaces][name][:inet][:prefix] = role[:inet][:prefix] - node.default_unless[:networking][:interfaces][name][:inet][:gateway] = role[:inet][:gateway] - node.default_unless[:networking][:interfaces][name][:inet][:routes] = role[:inet][:routes] - node.default_unless[:networking][:interfaces][name][:inet][:rules] = role[:inet][:rules] - end + parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" - if interface[:inet6] && role[:inet6] - node.default_unless[:networking][:interfaces][name][:inet6][:prefix] = role[:inet6][:prefix] - node.default_unless[:networking][:interfaces][name][:inet6][:gateway] = role[:inet6][:gateway] - node.default_unless[:networking][:interfaces][name][:inet6][:routes] = role[:inet6][:routes] - node.default_unless[:networking][:interfaces][name][:inet6][:rules] = role[:inet6][:rules] - end + node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface + node.default_unless[:networking][:interfaces][parent][:vlans] = [] - node.default_unless[:networking][:interfaces][name][:metric] = role[:metric] - node.default_unless[:networking][:interfaces][name][:zone] = role[:zone] + node.default[:networking][:interfaces][parent][:vlans] << vlan_id end node[:networking][:interfaces].each do |_, interface| @@ -103,6 +82,8 @@ node[:networking][:interfaces].each do |_, interface| notifies :run, "notify_group[networkctl-reload]" end elsif interface[:interface] =~ /^bond\d+$/ + next unless interface[:bond] + template "/etc/systemd/network/10-#{interface[:interface]}.netdev" do source "bond.netdev.erb" owner "root" @@ -335,8 +316,10 @@ link "/etc/resolv.conf" do to "../run/systemd/resolve/stub-resolv.conf" end +package "ruby" + gem_package "dbus-systemd" do - gem_binary node[:ruby][:gem] + gem_binary node[:ruby][:system_gem] end prometheus_exporter "resolved" do diff --git a/cookbooks/networking/templates/default/network.erb b/cookbooks/networking/templates/default/network.erb index 7f38de3c5..f837c65a5 100644 --- a/cookbooks/networking/templates/default/network.erb +++ b/cookbooks/networking/templates/default/network.erb @@ -91,6 +91,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> @@ -107,6 +110,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> diff --git a/cookbooks/openssh/recipes/default.rb b/cookbooks/openssh/recipes/default.rb index 8b57aaaef..98e2fa9f2 100644 --- a/cookbooks/openssh/recipes/default.rb +++ b/cookbooks/openssh/recipes/default.rb @@ -43,7 +43,7 @@ hosts = search(:node, "networking:interfaces").sort_by { |n| n[:hostname] }.coll names = [name] unless node.interfaces(:role => :internal).empty? - names.unshift("#{name}.#{node[:networking][:roles][:external][:zone]}.openstreetmap.org") + names.unshift("#{name}.#{node[:networking][:search].first}") end unless node.interfaces(:role => :external).empty? diff --git a/cookbooks/openssh/templates/default/ssh_known_hosts.erb b/cookbooks/openssh/templates/default/ssh_known_hosts.erb index c2d996a75..e0f173c06 100644 --- a/cookbooks/openssh/templates/default/ssh_known_hosts.erb +++ b/cookbooks/openssh/templates/default/ssh_known_hosts.erb @@ -20,11 +20,12 @@ yevaud.oob.openstreetmap.org,yevaud.oob,10.0.1.15 ssh-rsa AAAAB3NzaC1yc2EAAAABIw switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCbUecW/591rGCYtkdVTUgAKTBc+lEhraeMCau7CHqMkkUn+/pe68toDySdTaO8jua1q1asuEWH3MZ2AGAPSF+xhrqJLLPQmm0WdEAQfZChxQUoqCPqlWAuM5u+bAc3ntKjw2v2myoGGAqjL27QkZDP8qPdxvKcbm2YpPVOHP56jsyp/9BjVhIpWVmMhkOAfBSjdKU+uP+NueHrJ/AueoqucQE0txeo5Mmw7GlRuLTsMNDU3Khlhjs/2Q0QGiMXV6hQxj5AgnaDZMdSRbp8lXlctDZmipr7jVk3TL+knzySih5wJ7wmeZimW8Dhcxk6HBo2lGgPGdtQgttCF5ZUGcfF switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFPNVsMuGWMe2OP9AIg0hqZLSJP8zyNHWqYrXd5Mnfr12XdhzWEAKLTD14ZnJcz/GBaE0ad3IjT+hVoS/GVtxpQ= switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA615MNtLzOv6qPfFsD0I6Pd0itshpHLAVImjt1ME4SM -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2oeoe4olettjDauLuRiXRQVg5k6EO+PFfS9Ho3OzlOeYr1+h1SSPLkFtr20yOKcGXQcmYLjTbzvt/sbz2JUbfJHc2n7tMvQKK5ClVxLwqt2jpfSepGbG2KjG6vwxpkcDMH0bquzr/8ehpYmBwh3m8EARgcVNYoVL6J5OV4ILAtjNEllsrB71gitrarC8iqslRLVMjane9YKa7z0AGkwcT9w4qbcyuQ24rJF0Xo/IcraxVAYeX0Xtfx9rD3Zt1kHawaKYQdJM7RZvTtWcgCzHt738lFZ15Qj3at6vxxGpnduZiuHLB3PCW9yP4CFHm3Qs3TUyh3VqrNVFFdFvUXnP+DR3QK2E7TpViagZmXb+2AeCaLtrJ+1h0NEmQmliB4Zm4XKPsdS7k6Ul43pYQ5i1ipwZBNvhJrByQJ6UkpIjrwUlQBxXcEWGZaSL0sqqDQIYPizXZWOh7pQay2LxnHUVp0MZT30iVLYjptu9PcJ6CBwoKzhnVHPh7SON4uzDyMrk= -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFLujNadDWF9OqF//izQ3/jm4zElWb2nMjfpeiUVCYh/Jrsl07c1bRJlnSmoxL2H0xeHJXQh1E4PPy4o/5N4GfE= -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINxBm8YPJNtUIDh+0YWKY/5XKv8QKQBtF3ecqQcxIQNJ +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTgHga8l3mOpydchsZsvlaCJpxHoUV5dYBzK8m3iOedqd1UBS9KbHEfgFOvXcd3cmOnQT5dM1YXCQ74WB9hI5Wbkz+oHxWq+ZJeHOsXVIqxFzgxsI8e8gxJve7VyzxR52yiStBdLODhwwMmRPIH8AzzR93Fzi5AAQrGoHoqi/R0A/l6yTI8YnjJ2WaKzflId0YMyjR2KhOj+jCgCZIJR6DqfVpZXERMAtdVUY78Ub0E8iYyWg24gI176l4jCezmaTZ99tWo41mGXax9zXizQr4jx4yIxzRMhqAoJk8oDBkdhxudRUL2eWvBs3v1kcY3Ywtxi+wQIqClvQgBkWrHe4Jujjl8EEFuGHh6156Al7WqEnC9EcENA5Lk/IM0lzDbeT+WTyO/qU8cu035hmPMydYBHjqzo5L6yDwFXEAtJpHlr8QQDzEwgtK9bXc453cQDEJW4JWjP7UlsTlbflSBGQrY5FpI8D+MgnDYNld2kD4Fxzomf4ne+M0w19jrtNMJuE= +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPkUAdFYwolSkUs7d1k+mRVBCV7WRrKsebj8RbjLqPCmEQo9ABSy2iAuYOYgkkqlgmB8F8OHsnNtmGzx7smCwAU= +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKVdlLkjFUDExO7YCixEWvaKHGS0IQHn2nBrkMjGTwRx +ats1.ams.openstreetmap.org,ats1.ams,10.0.48.103 ssh-rsa AAAAB3NzaC1yc2EAAAACAQEAAAEBAKnx+hwj8Am4DuYXZLK1whHqOjW4Bbs/KrLqFokoZgJxzp1g+6sOAzXPy8w3dYyHBJaXCmr3RisIhG7fj0pITkywhU8ygiaEHDzPFyGqxVYe1oI3ayE47hC14Jhu33p7F+pY1xEVUa2H3Xu4F8toHnmuIjLAJ9phaEVvnjLl5n47dM1VfB5KB4NvuDejBthaZpgbALainUuKaqUL2Wgb0LxbDFRTQ+r6uUY4SEFnfqwveovdBL/edd+14/yRob0/r6zlubi5PhvMH1rCaE7flFNr61LItz06VAs+vZpLTNTIFXHqAtCjIA6e8VvvOn8wwmjxXKLsUOFZipvW9Uc9kTk= faffy.oob.openstreetmap.org,faffy.oob,10.0.49.3 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuL6kBDuK+/xj40jp4ZP9HVoH1enU1W9MWO4Q0cxlD9sjHaz/mIbLhtPxElrw807QLsUdOx1IeD0jCLi9XAYYEF2xFFZ9DIP+qGp6hQ3XJrmOVgbDuckPjdvJWAxC5TVaWqwvHyp02biJCfSbOzsmjm2cOp03Cqq18HJOCfA3AxQ/zSQDSVemFoFPVeM67M9NgM64z1idPAvMVtW8Kd+unU/oE7V+Jil5TR5vwK+kVKEixxMtZfkJn0bNKVFWX5sJYmKWze+b95O+iQlsxmQW/p7UVM/3krKdBBT8+9b6UOv8TN0Yhopp86aXEgtvUOOyyimjqO3V5ogRMGj3b9G+9 -vhagar.oob.openstreetmap.org,10.0.49.5 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJaXI71RCPjbni+WxBnOf2CzkLi0H5ARG1RxevLO5smM/9LbXVAVgcZRnI3egmbaJF7r6queHtrQq5y27+Rz9OEoJ/DWMS/qaGWMqj1f7LDv+H44juKAxmayOYkLJUnXN/AYITjpNSfLy0kuSo9cGpHVKjQZVFGv92b0c1CPrQ0WJ3Q8Y7ERc77b2PBQmxWow/3RHv6SoeT8riCfoU5Mnuld2aNjlJlEglf79QB8r7Xonx3TDfcr0+bGbvJNhYoMPPDXFC96wCpjvIDQu09BEM91/zP/VRE3E+bvIp0WRugU6vUYAAIcKuyQpON5Fs1kWaY4AZ6/4+Qdql5B8z8twz +vhagar.oob.openstreetmap.org,vhagar.oob,10.0.49.5 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJaXI71RCPjbni+WxBnOf2CzkLi0H5ARG1RxevLO5smM/9LbXVAVgcZRnI3egmbaJF7r6queHtrQq5y27+Rz9OEoJ/DWMS/qaGWMqj1f7LDv+H44juKAxmayOYkLJUnXN/AYITjpNSfLy0kuSo9cGpHVKjQZVFGv92b0c1CPrQ0WJ3Q8Y7ERc77b2PBQmxWow/3RHv6SoeT8riCfoU5Mnuld2aNjlJlEglf79QB8r7Xonx3TDfcr0+bGbvJNhYoMPPDXFC96wCpjvIDQu09BEM91/zP/VRE3E+bvIp0WRugU6vUYAAIcKuyQpON5Fs1kWaY4AZ6/4+Qdql5B8z8twz dulcy.oob.openstreetmap.org,dulcy.oob,10.0.49.9 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgmlwXGPIPDqoMz4s5edr+G4iuBnCYcFXjY7Et9jG/ezI6aIKd9lEOXvnHJX5hrC21aqd06dhyMLwZN+eSjJhXNaLtYmm5P442H2ZnGca1KNSpF7yRVhn0eRpX39xWK3biVfMw64mzvV3636C1adCkgBwuaqCvz1EHm/KRrySfuETFLU= spike-06.oob.openstreetmap.org,spike-06.oob,10.0.49.6 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDwQVb/cYgKp21vw6+jKr5TLAD77oKunbFbjvwPL34iLRDqdJcEOdJBV1A63ZDUWSWjDUrTa0kyneoiYdD876EpuspZ4bPgmiHBvug5NR3usxa8PAsia1K+fONAGi3+s96H6Us273KHNH5QwyBNDyYaDDjIECkydU4bQT0FSX0j32eAHI62ZO/H82fUyL20PIK/nCUaJUd+5iXgcTfjP+eL66ghgPCgXGF99/w2+3EUxP2IN9J2yI57H/rYn3rj+NdZyQuzc0TAToT341cYh2+a8R1Sb/DEFiMZFE2hJ2NeVEaBYxPSlFfhRtlB6ArXg5QL7e7UScQh/ZHLU26A1ZaB spike-07.oob.openstreetmap.org,spike-07.oob,10.0.49.7 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxLxAbMPZw+LgzB8JmTLLxqc6CZXYA3Wo18RM02+EnmtRX6AAf/VFb7E/VpgHfZecwnFK7u21R47+fFdHCb8EDYGeWbOYoEOYMoDh26H8/aFPHbldgG0xs9EwfTryWB7iZ2sD9nLv+nBpDkFXVq53kscmDlUKQrvUyj/zR55xGxcq0ruu3w05ZeGx5I6HRY0xeK+6H4s6sZtF24Zm3CttQ5M7ADgWffE35ZwA9tEdioM3J3c5EcV4HMdhl7wame6pID9+FpDEtcts5z5sfW9Y0yx3FHrXCxRWxHRbyFG9/+4RNwKJBRZrdb+VSLlrYVUFLEfb1AhOxr8ZfSsJU1V0V diff --git a/cookbooks/overpass/templates/default/totp-filter.erb b/cookbooks/overpass/templates/default/totp-filter.erb index 8245f2ae3..78160cfe9 100644 --- a/cookbooks/overpass/templates/default/totp-filter.erb +++ b/cookbooks/overpass/templates/default/totp-filter.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "cgi" require "rotp" diff --git a/cookbooks/oxidized/recipes/default.rb b/cookbooks/oxidized/recipes/default.rb index fce3126d3..1840c2af3 100644 --- a/cookbooks/oxidized/recipes/default.rb +++ b/cookbooks/oxidized/recipes/default.rb @@ -130,9 +130,15 @@ git "/var/lib/oxidized/configs.git" do group "oxidized" end +bundle_config "/opt/oxidized/daemon" do + user "oxidized" + group "oxidized" + settings "deployment" => "true", + "build.rugged" => "--with-ssh" +end + bundle_install "/opt/oxidized/daemon" do action :nothing - options "--deployment" user "oxidized" group "oxidized" notifies :restart, "service[oxidized]" @@ -150,6 +156,7 @@ systemd_service "oxidized" do "OXIDIZED_LOGS" => "/var/log/oxidized" nice 10 sandbox :enable_network => true + restrict_address_families "AF_NETLINK" read_write_paths ["/run/oxidized", "/var/lib/oxidized", "/var/log/oxidized"] restart "on-failure" notifies :restart, "service[oxidized]" diff --git a/cookbooks/passenger/templates/default/passenger.conf.erb b/cookbooks/passenger/templates/default/passenger.conf.erb index 1212f7588..5533f8deb 100644 --- a/cookbooks/passenger/templates/default/passenger.conf.erb +++ b/cookbooks/passenger/templates/default/passenger.conf.erb @@ -2,8 +2,9 @@ PassengerRoot /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini - PassengerDefaultRuby /usr/bin/ruby + PassengerDefaultRuby <%= node[:ruby][:interpreter] %> PassengerRuby /usr/local/bin/passenger-ruby + PassengerPreloadBundler on PassengerMaxPoolSize <%= node[:passenger][:max_pool_size] %> PassengerPoolIdleTime <%= node[:passenger][:pool_idle_time] %> PassengerFriendlyErrorPages off diff --git a/cookbooks/passenger/templates/default/ruby.erb b/cookbooks/passenger/templates/default/ruby.erb index f45fd210e..2af922032 100644 --- a/cookbooks/passenger/templates/default/ruby.erb +++ b/cookbooks/passenger/templates/default/ruby.erb @@ -4,4 +4,4 @@ export RUBY_GC_HEAP_INIT_SLOTS=500000 export RUBY_GC_HEAP_FREE_SLOTS=100000 export RUBY_GC_MALLOC_LIMIT=50000000 -exec /usr/bin/ruby<%= node[:ruby][:version] %> "$@" +exec <%= node[:ruby][:interpreter] %> "$@" diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb index d719d11ae..f7552ddf2 100644 --- a/cookbooks/planet/recipes/replication.rb +++ b/cookbooks/planet/recipes/replication.rb @@ -32,14 +32,18 @@ db_passwords = data_bag_item("db", "passwords") package %w[ postgresql-client - ruby-libxml make gcc libc6-dev libpq-dev + libxml2-dev osmdbt ] +gem_package "libxml-ruby" do + gem_binary node[:ruby][:gem] +end + gem_package "pg" do gem_binary node[:ruby][:gem] end @@ -91,6 +95,13 @@ template "/usr/local/bin/users-deleted" do mode "755" end +template "/usr/local/bin/replicate-changesets" do + source "replicate-changesets.erb" + owner "root" + group "root" + mode "755" +end + ## Published deleted users directory remote_directory "/store/planet/users_deleted" do diff --git a/cookbooks/planet/templates/default/planet-file-cleanup.erb b/cookbooks/planet/templates/default/planet-file-cleanup.erb index 52ce68623..1df8eb595 100644 --- a/cookbooks/planet/templates/default/planet-file-cleanup.erb +++ b/cookbooks/planet/templates/default/planet-file-cleanup.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require 'date' require 'optparse' diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/templates/default/replicate-changesets.erb similarity index 99% rename from cookbooks/planet/files/default/replication-bin/replicate-changesets rename to cookbooks/planet/templates/default/replicate-changesets.erb index bb4544f68..e2e5aca68 100755 --- a/cookbooks/planet/files/default/replication-bin/replicate-changesets +++ b/cookbooks/planet/templates/default/replicate-changesets.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "rubygems" require "pg" diff --git a/cookbooks/postgresql/recipes/default.rb b/cookbooks/postgresql/recipes/default.rb index 742941ee9..eae492f16 100644 --- a/cookbooks/postgresql/recipes/default.rb +++ b/cookbooks/postgresql/recipes/default.rb @@ -124,24 +124,20 @@ clusters.each do |name, details| not_if { ::File.exist?("/var/lib/postgresql/#{name}/standby.signal") } end - exporter_options = %w[ - --collector.database_wraparound - --collector.long_running_transactions - --collector.process_idle - --collector.stat_activity_autovacuum - --collector.stat_wal_receiver - --collector.statio_user_indexes - ] - - exporter_options << "--no-collector.stat_bgwriter" if details[:version] >= 17 - prometheus_exporter "postgres" do port 10000 + details[:port].to_i service "postgres-#{prometheus_suffix}" labels "cluster" => name scrape_interval "1m" scrape_timeout "1m" - options exporter_options + options %w[ + --collector.database_wraparound + --collector.long_running_transactions + --collector.process_idle + --collector.stat_activity_autovacuum + --collector.stat_wal_receiver + --collector.statio_user_indexes + ] environment "DATA_SOURCE_NAME" => "postgres:///#{prometheus_database}?host=/run/postgresql&port=#{details[:port]}&user=prometheus&password=#{passwords['prometheus']}" restrict_address_families "AF_UNIX" subscribes :restart, "template[/etc/prometheus/exporters/postgres_queries.yml]" diff --git a/cookbooks/prometheus/metadata.rb b/cookbooks/prometheus/metadata.rb index 30f20802d..6148979b9 100644 --- a/cookbooks/prometheus/metadata.rb +++ b/cookbooks/prometheus/metadata.rb @@ -12,4 +12,3 @@ depends "awscli" depends "git" depends "hardware" depends "networking" -depends "ruby" diff --git a/cookbooks/prometheus/recipes/default.rb b/cookbooks/prometheus/recipes/default.rb index c6b609b06..df3ec2ea5 100644 --- a/cookbooks/prometheus/recipes/default.rb +++ b/cookbooks/prometheus/recipes/default.rb @@ -19,7 +19,8 @@ include_recipe "git" include_recipe "networking" -include_recipe "ruby" + +package "ruby" if node.internal_ipaddress node.default[:prometheus][:mode] = "internal" diff --git a/cookbooks/prometheus/resources/collector.rb b/cookbooks/prometheus/resources/collector.rb index ce68a7e5c..4171b874d 100644 --- a/cookbooks/prometheus/resources/collector.rb +++ b/cookbooks/prometheus/resources/collector.rb @@ -27,6 +27,7 @@ property :user, :kind_of => String property :path, :kind_of => String property :options, :kind_of => [String, Array] property :environment, :kind_of => Hash, :default => {} +property :working_directory, String property :proc_subset, String property :capability_bounding_set, [String, Array] property :private_devices, [true, false] @@ -42,6 +43,7 @@ action :create do dynamic_user new_resource.user.nil? group "adm" environment new_resource.environment + working_directory new_resource.working_directory standard_output "file:/var/lib/prometheus/node-exporter/#{new_resource.collector}.new" standard_error "journal" exec_start "#{executable_path} #{executable_options}" diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb index 305afbd90..27cb72e47 100644 --- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb +++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb @@ -828,7 +828,7 @@ groups: - name: web rules: - alert: web error rate - expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 and sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) > 0.01 + expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 and sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) > 0.05 for: 5m labels: alertgroup: web diff --git a/cookbooks/ruby/attributes/default.rb b/cookbooks/ruby/attributes/default.rb index eefaf2247..39ac754e5 100644 --- a/cookbooks/ruby/attributes/default.rb +++ b/cookbooks/ruby/attributes/default.rb @@ -1,9 +1,28 @@ -default[:ruby][:version] = if platform?("debian") - "3.1" - elsif node[:lsb][:release].to_f < 22.04 - "2.7" - else - "3.0" - end -default[:ruby][:gem] = "/usr/bin/gem#{node[:ruby][:version]}" -default[:ruby][:bundle] = "/usr/bin/bundle#{node[:ruby][:version]}" +default[:ruby][:fullstaq] = true + +default[:ruby][:system_version] = if platform?("debian") + "3.1" + elsif node[:lsb][:release].to_f < 22.04 + "2.7" + else + "3.0" + end +default[:ruby][:system_interpreter] = "/usr/bin/ruby#{node[:ruby][:system_version]}" +default[:ruby][:system_gem] = "/usr/bin/gem#{node[:ruby][:system_version]}" +default[:ruby][:system_bundle] = "/usr/bin/bundle#{node[:ruby][:system_version]}" + +if node[:ruby][:fullstaq] + + default[:ruby][:version] = "3.4" + default[:ruby][:interpreter] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/ruby" + default[:ruby][:gem] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/gem" + default[:ruby][:bundle] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/bundle" + +else + + default[:ruby][:version] = node[:ruby][:system_version] + default[:ruby][:interpreter] = node[:ruby][:system_interpreter] + default[:ruby][:gem] = node[:ruby][:system_gem] + default[:ruby][:bundle] = node[:ruby][:system_bundle] + +end diff --git a/cookbooks/ruby/metadata.rb b/cookbooks/ruby/metadata.rb index d8f2eac0e..6c014bdfe 100644 --- a/cookbooks/ruby/metadata.rb +++ b/cookbooks/ruby/metadata.rb @@ -6,3 +6,4 @@ description "Installs and configures ruby" version "1.0.0" supports "ubuntu" +depends "apt" diff --git a/cookbooks/ruby/recipes/default.rb b/cookbooks/ruby/recipes/default.rb index a2833aefa..7301e372c 100644 --- a/cookbooks/ruby/recipes/default.rb +++ b/cookbooks/ruby/recipes/default.rb @@ -19,23 +19,29 @@ ruby_version = node[:ruby][:version] -package %W[ - ruby - ruby#{ruby_version} - ruby - ruby#{ruby_version}-dev -] - -gem_package "bundler#{ruby_version}-1" do - package_name "bundler" - version "~> 1.17.3" - gem_binary node[:ruby][:gem] - options "--format-executable" -end +if node[:ruby][:fullstaq] + + include_recipe "apt::fullstaq-ruby" + + package %W[ + fullstaq-ruby-common + fullstaq-ruby-#{ruby_version}-jemalloc + ] + + %w[bundle bundler erb gem irb racc rake rbs rdbg rdoc ri ruby syntax_suggest typeproc].each do |command| + link "/usr/local/bin/#{command}" do + to "/usr/lib/fullstaq-ruby/versions/#{ruby_version}-jemalloc/bin/#{command}" + owner "root" + group "root" + end + end + +else + + package %W[ + ruby + ruby-dev + ruby-bundler + ] -gem_package "bundler#{ruby_version}-2" do - package_name "bundler" - version "~> 2.3.16" - gem_binary node[:ruby][:gem] - options "--format-executable" end diff --git a/cookbooks/ruby/resources/bundle_config.rb b/cookbooks/ruby/resources/bundle_config.rb new file mode 100644 index 000000000..16902682f --- /dev/null +++ b/cookbooks/ruby/resources/bundle_config.rb @@ -0,0 +1,57 @@ +# +# Cookbook:: ruby +# Resource:: bundle_config +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +resource_name :bundle_config +provides :bundle_config + +unified_mode true + +default_action :create + +property :directory, :kind_of => String, :name_property => true +property :user, :kind_of => String +property :group, :kind_of => String +property :settings, :kind_of => Hash + +load_current_value do |new_resource| + current_settings = shell_out!("#{node[:ruby][:bundle]} config list --parseable", :cwd => new_resource.directory).stdout.split("\n").map do |line| + line.split("=") + end.to_h + + settings current_settings +end + +action :create do + converge_if_changed :settings do + new_resource.settings.each do |name, value| + execute "bundle-config-set-#{name}" do + command "#{bundle_command} config set --local #{name} #{value}" + cwd new_resource.directory + user new_resource.user + group new_resource.group + end + end + end +end + +action_class do + def bundle_command + node[:ruby][:bundle] + end +end diff --git a/cookbooks/ruby/resources/bundle_install.rb b/cookbooks/ruby/resources/bundle_install.rb index 897bde5fe..9facccc27 100644 --- a/cookbooks/ruby/resources/bundle_install.rb +++ b/cookbooks/ruby/resources/bundle_install.rb @@ -47,6 +47,9 @@ action_class do end def after_created - subscribes :run, "gem_package[bundler#{node[:ruby][:version]}-1]" - subscribes :run, "gem_package[bundler#{node[:ruby][:version]}-2]" + if node[:ruby][:fullstaq] + subscribes :run, "package[fullstaq-ruby-#{node[:ruby][:version]}-jemalloc]" + else + subscribes :run, "package[ruby-bundler]" + end end diff --git a/cookbooks/serverinfo/attributes/default.rb b/cookbooks/serverinfo/attributes/default.rb new file mode 100644 index 000000000..bd45555d1 --- /dev/null +++ b/cookbooks/serverinfo/attributes/default.rb @@ -0,0 +1 @@ +default[:accounts][:users][:serverinfo][:status] = :role diff --git a/cookbooks/serverinfo/metadata.rb b/cookbooks/serverinfo/metadata.rb index 55d69f066..48a196ad6 100644 --- a/cookbooks/serverinfo/metadata.rb +++ b/cookbooks/serverinfo/metadata.rb @@ -6,6 +6,7 @@ description "Configures server-info web site" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" depends "git" depends "ruby" diff --git a/cookbooks/serverinfo/recipes/default.rb b/cookbooks/serverinfo/recipes/default.rb index 2bbc31e5b..c455ddc12 100644 --- a/cookbooks/serverinfo/recipes/default.rb +++ b/cookbooks/serverinfo/recipes/default.rb @@ -17,6 +17,7 @@ # limitations under the License. # +include_recipe "accounts" include_recipe "apache" include_recipe "git" include_recipe "ruby" @@ -30,12 +31,18 @@ package %w[ pkg-config ] +directory "/srv/hardware.openstreetmap.org" do + owner "serverinfo" + group "serverinfo" + mode "755" +end + git "/srv/hardware.openstreetmap.org" do action :sync repository "https://github.com/osmfoundation/osmf-server-info.git" depth 1 - user "root" - group "root" + user "serverinfo" + group "serverinfo" notifies :run, "bundle_install[/srv/hardware.openstreetmap.org]" end @@ -45,52 +52,43 @@ roles = { :rows => search(:role, "*:*") } file "/srv/hardware.openstreetmap.org/_data/nodes.json" do content nodes.to_json mode "644" - owner "root" - group "root" + owner "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" + sensitive true end file "/srv/hardware.openstreetmap.org/_data/roles.json" do content roles.to_json mode "644" - owner "root" - group "root" + owner "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end -directory "/srv/hardware.openstreetmap.org/_site" do - mode "755" - owner "nobody" - group "nogroup" -end - -directory "/srv/hardware.openstreetmap.org/vendor" do +bundle_config "/srv/hardware.openstreetmap.org" do action :create - owner "nobody" - group "nogroup" - notifies :run, "bundle_install[/srv/hardware.openstreetmap.org]", :immediately + user "serverinfo" + group "serverinfo" + settings "deployment" => "true", + "without" => "development:test", + "jobs" => node.cpu_cores.to_s + notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end bundle_install "/srv/hardware.openstreetmap.org" do action :nothing - user "nobody" - group "nogroup" - environment "BUNDLE_FROZEN" => "true", - "BUNDLE_WITHOUT" => "development:test", - "BUNDLE_PATH" => "vendor/bundle", - "BUNDLE_DEPLOYMENT" => "1", - "BUNDLE_JOBS" => node.cpu_cores.to_s + user "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end bundle_exec "/srv/hardware.openstreetmap.org" do action :nothing command "jekyll build --trace --disable-disk-cache --baseurl=https://hardware.openstreetmap.org" - user "nobody" - group "nogroup" - environment "LANG" => "C.UTF-8", - "BUNDLE_PATH" => "vendor/bundle", - "BUNDLE_DEPLOYMENT" => "1" + user "serverinfo" + group "serverinfo" + environment "LANG" => "C.UTF-8" end ssl_certificate "hardware.openstreetmap.org" do diff --git a/cookbooks/stateofthemap/recipes/container.rb b/cookbooks/stateofthemap/recipes/container.rb index 3b5bd38b9..8e97accc0 100644 --- a/cookbooks/stateofthemap/recipes/container.rb +++ b/cookbooks/stateofthemap/recipes/container.rb @@ -24,7 +24,7 @@ podman_site "stateofthemap.org" do aliases ["www.stateofthemap.org", "stateofthemap.com", "www.stateofthemap.com", "sotm.org", "www.sotm.org"] end -%w[2007 2008 2009 2013 2016 2017 2018 2019 2020 2021 2022 2024].each do |year| +%w[2007 2008 2009 2013 2016 2017 2018 2019 2020 2021 2022 2024 2025].each do |year| podman_site "#{year}.stateofthemap.org" do image "ghcr.io/openstreetmap/stateofthemap-#{year}:latest" aliases ["#{year}.stateofthemap.com", "#{year}.sotm.org"] diff --git a/cookbooks/systemd/resources/service.rb b/cookbooks/systemd/resources/service.rb index 11ca0e28c..8b380c306 100644 --- a/cookbooks/systemd/resources/service.rb +++ b/cookbooks/systemd/resources/service.rb @@ -36,6 +36,7 @@ property :notify_access, String, :is => %w[none main exec all] property :limit_nofile, Integer property :limit_as, [Integer, String] property :limit_cpu, [Integer, String] +property :limit_core, [Integer, String] property :memory_low, [Integer, String] property :memory_high, [Integer, String] property :memory_max, [Integer, String] diff --git a/cookbooks/systemd/templates/default/service.erb b/cookbooks/systemd/templates/default/service.erb index 92d70de85..eee220691 100644 --- a/cookbooks/systemd/templates/default/service.erb +++ b/cookbooks/systemd/templates/default/service.erb @@ -45,6 +45,9 @@ LimitAS=<%= @limit_as %> <% if @limit_cpu -%> LimitCPU=<%= @limit_cpu %> <% end -%> +<% if @limit_core -%> +LimitCORE=<%= @limit_core %> +<% end -%> <% if @memory_low -%> MemoryLow=<%= @memory_low %> <% end -%> diff --git a/cookbooks/taginfo/recipes/default.rb b/cookbooks/taginfo/recipes/default.rb index e4a6da00f..ef3913c60 100644 --- a/cookbooks/taginfo/recipes/default.rb +++ b/cookbooks/taginfo/recipes/default.rb @@ -179,10 +179,17 @@ node[:taginfo][:sites].each do |site| notifies :restart, "service[apache2]" end + bundle_config "#{directory}/taginfo" do + user "taginfo" + group "taginfo" + settings "deployment" => "true", + "without" => "development:test" + end + bundle_install "#{directory}/taginfo" do action :nothing - user "root" - group "root" + user "taginfo" + group "taginfo" subscribes :run, "git[#{directory}/taginfo]" notifies :restart, "passenger_application[#{directory}/taginfo/web/public]" end @@ -225,7 +232,8 @@ node[:taginfo][:sites].each do |site| prometheus_collector "taginfo-#{site_name}" do interval "15m" user "taginfo" - path "#{directory}/taginfo/sources/metrics.rb" - options "#{directory}/data" + path node[:ruby][:bundle] + options "exec sources/metrics.rb #{directory}/data" + working_directory "#{directory}/taginfo" end end diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 13b897363..f969546af 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -159,6 +159,7 @@ end package %w[ python3-cairo python3-mapnik + python3-pil python3-pyotp python3-pyproj python3-setuptools diff --git a/cookbooks/tile/templates/default/expire-tiles.erb b/cookbooks/tile/templates/default/expire-tiles.erb index b3a790f6a..5bc478082 100644 --- a/cookbooks/tile/templates/default/expire-tiles.erb +++ b/cookbooks/tile/templates/default/expire-tiles.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> # DO NOT EDIT - This file is being maintained by Chef diff --git a/cookbooks/tile/templates/default/export.erb b/cookbooks/tile/templates/default/export.erb index 8fa4672b9..eb35b1313 100644 --- a/cookbooks/tile/templates/default/export.erb +++ b/cookbooks/tile/templates/default/export.erb @@ -5,6 +5,7 @@ import cairo import cgi import http.cookies import mapnik +import io import os import pyotp import pyproj @@ -13,6 +14,7 @@ import shutil import signal import sys import tempfile +from PIL import Image # Limit maximum CPU time # The Postscript output format can sometimes take hours @@ -40,6 +42,10 @@ def output_file(file): def file_size(file): return os.fstat(file.fileno()).st_size +# Routine to retrieve BytesIO payload length +def bytesio_size(bio): + return bio.getbuffer().nbytes + # Routine to report an error def output_error(message, status = "400 Bad Request"): print("Status: %s" % status) @@ -54,6 +60,105 @@ def output_error(message, status = "400 Bad Request"): print("") print("") +# Add a copyright notice for raster formats (PNG, JPEG, WEBP) +def add_copyright_notice_raster(image, map_width, map_height, format): + # Convert the Mapnik image to PNG and store it in a BytesIO object + png = image.tostring("png") + png_io = io.BytesIO(png) + + # Load the PNG data from the BytesIO object into a Cairo ImageSurface + surface = cairo.ImageSurface.create_from_png(png_io) + + add_copyright_notice_vector(surface, map_width, map_height) + + # Convert the Cairo surface to PNG in a BytesIO object + output_io = io.BytesIO() + surface.write_to_png(output_io) + + if format == "png": + return output_io + else: + # Open the output PNG image for conversion to other formats + img = Image.open(output_io) + img_io = io.BytesIO() + img.save(img_io, format=format) + return img_io + +# Add a copyright notice for vector formats (SVG, PDF, PS) +def add_copyright_notice_vector(surface, map_width, map_height): + context = cairo.Context(surface) + + # Set the font for the copyright notice + context.set_font_face(cairo.ToyFontFace("DejaVu")) + context.set_font_size(14) + + # Define the copyright text + text = "© OpenStreetMap contributors" + + text_extents = context.text_extents(text) + text_width = text_extents.width + text_height = text_extents.height + + x_margin = 10 + y_margin = 10 + + # Position the text at the bottom-right corner + x_position = map_width - text_width - x_margin + y_position = map_height - text_height - y_margin + + # Draw a white box just large enough to fit the text + context.set_source_rgba(1, 1, 1, 0.5) + context.rectangle(x_position - x_margin, y_position - y_margin, + text_width + 2 * x_margin, text_height + 2 * y_margin) + context.fill_preserve() + + context.set_source_rgb(0, 0, 0) # Black color for the text + context.move_to(x_position - x_margin / 2, y_position + y_margin) + context.show_text(text) + +# Render and output map for raster formats (PNG, JPEG, WEBP) +def render_and_output_image(map, format): + image = mapnik.Image(map.width, map.height) + mapnik.render(map, image) + + bytes_io = add_copyright_notice_raster(image, map.width, map.height, format) + + if format == "png": + output_headers("image/png", "map.png", bytesio_size(bytes_io)) + elif format == "jpeg": + output_headers("image/jpeg", "map.jpg", bytesio_size(bytes_io)) + elif format == "webp": + output_headers("image/webp", "map.webp", bytesio_size(bytes_io)) + + output_file(bytes_io) + +# Render and output map for vector formats (SVG, PDF, PS) +def render_and_output_vector(map, format): + with tempfile.NamedTemporaryFile(prefix="export") as file: + if format == "svg": + surface = cairo.SVGSurface(file.name, map.width, map.height) + surface.restrict_to_version(cairo.SVG_VERSION_1_2) + elif format == "pdf": + surface = cairo.PDFSurface(file.name, map.width, map.height) + elif format == "ps": + surface = cairo.PSSurface(file.name, map.width, map.height) + + mapnik.render(map, surface) + + add_copyright_notice_vector(surface, map.width, map.height) + + surface.finish() + + if format == "svg": + output_headers("image/svg+xml", "map.svg", file_size(file)) + elif format == "pdf": + output_headers("application/pdf", "map.pdf", file_size(file)) + elif format == "ps": + output_headers("application/postscript", "map.ps", file_size(file)) + + output_file(file) + + # Create TOTP token validator totp = pyotp.TOTP('<%= @totp_key %>', interval = 3600) @@ -149,42 +254,13 @@ else: # Render the map if pid == 0: - if form.getvalue("format") == "png": - image = mapnik.Image(map.width, map.height) - mapnik.render(map, image) - png = image.tostring("png") - output_headers("image/png", "map.png", len(png)) - sys.stdout.buffer.write(png) - elif form.getvalue("format") == "jpeg": - image = mapnik.Image(map.width, map.height) - mapnik.render(map, image) - jpeg = image.tostring("jpeg") - output_headers("image/jpeg", "map.jpg", len(jpeg)) - sys.stdout.buffer.write(jpeg) - elif form.getvalue("format") == "svg": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.SVGSurface(file.name, map.width, map.height) - surface.restrict_to_version(cairo.SVG_VERSION_1_2) - mapnik.render(map, surface) - surface.finish() - output_headers("image/svg+xml", "map.svg", file_size(file)) - output_file(file) - elif form.getvalue("format") == "pdf": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.PDFSurface(file.name, map.width, map.height) - mapnik.render(map, surface) - surface.finish() - output_headers("application/pdf", "map.pdf", file_size(file)) - output_file(file) - elif form.getvalue("format") == "ps": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.PSSurface(file.name, map.width, map.height) - mapnik.render(map, surface) - surface.finish() - output_headers("application/postscript", "map.ps", file_size(file)) - output_file(file) + format = form.getvalue("format") + if format in ["png", "jpeg", "webp"]: + render_and_output_image(map, format) + elif format in ["svg", "pdf", "ps"]: + render_and_output_vector(map, format) else: - output_error("Unknown format '%s'" % form.getvalue("format")) + output_error("Unknown format") else: pid, status = os.waitpid(pid, 0) if status & 0xff == signal.SIGXCPU: diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb index b5e0933a6..f72621213 100644 --- a/cookbooks/vectortile/attributes/default.rb +++ b/cookbooks/vectortile/attributes/default.rb @@ -4,10 +4,12 @@ default[:vectortile][:database][:nodes_store] = :flat default[:vectortile][:serve][:threads] = node.cpu_cores default[:vectortile][:serve][:mode] = :live default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute" -default[:vectortile][:replication][:status] = :enabled -default[:vectortile][:replication][:tileupdate] = :enabled +default[:vectortile][:replication][:enabled] = true +default[:vectortile][:replication][:tileupdate] = true default[:vectortile][:replication][:threads] = node.cpu_cores +default[:vectortile][:tilekiln][:version] = "0.7.1" + default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] default[:postgresql][:monitor_database] = "tiles" # As an absolute worst case, the server might have the serving, update, and a manual generation process going on. diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb index a8a78044c..0f4041565 100644 --- a/cookbooks/vectortile/recipes/default.rb +++ b/cookbooks/vectortile/recipes/default.rb @@ -86,7 +86,7 @@ shortbread_config = "#{style_directory}/shortbread.yaml" themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark" git themepark_directory do repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git" - revision "7efc00db433fa7d2c49db00c32ffd27d8ced35f4" + revision "444bfbda82dea2899e77ac7f0e88ddf7f62c3b45" user "tileupdate" group "tileupdate" end @@ -100,7 +100,7 @@ end python_package "tilekiln" do python_virtualenv tilekiln_directory python_version "3" - version "0.6.5" + version node[:vectortile][:tilekiln][:version] end template "/srv/vector.openstreetmap.org/html/index.html" do @@ -246,19 +246,21 @@ execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do end template "/usr/local/bin/vector-update" do - source node[:vectortile][:replication][:tileupdate] == :enabled ? "vector-update-tile.erb" : "vector-update-notile.erb" + source node[:vectortile][:replication][:tileupdate] ? "vector-update-tile.erb" : "vector-update-notile.erb" owner "root" group "root" mode "755" variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender" end +rerender_layers = %w[addresses boundaries bridges buildings land pois public_transport sites street_polygons streets water_lines_labels water_lines water_polygons].join(" ") + template "/usr/local/bin/tiles-rerender" do source "tiles-rerender.erb" owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4 + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4, :layers => "#{rerender_layers}" end systemd_service "replicate" do @@ -279,7 +281,7 @@ systemd_timer "replicate" do accuracy_sec 5 end -if node[:vectortile][:replication][:status] == :enabled +if node[:vectortile][:replication][:enabled] service "replicate.timer" do action [:enable, :start] end diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb index 6c01b27f6..d1ca4bdab 100644 --- a/cookbooks/vectortile/templates/default/tiles-rerender.erb +++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb @@ -2,9 +2,12 @@ set -eu cd "<%= @expiry_dir %>" - +# Only expire z10+ tiles +rm -f z?-*.txt wc -l z*.txt -cat z*.txt | <%= @tilekiln_bin %> generate tiles \ +(for layer in <%= @layers %>; do + find . -type f -name "z*-${layer}.txt" -execdir sed -e "s/\$/,${layer}/" {} \; +done) | <%= @tilekiln_bin %> generate layers \ --source-dbname "<%= @source_database %>" \ --storage-dbname "<%= @storage_database %>" \ --num-threads "<%= node[:vectortile][:replication][:threads] %>" \ diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb index ba708c9b8..a356cbb10 100644 --- a/cookbooks/vectortile/templates/default/vector-update-tile.erb +++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb @@ -8,6 +8,8 @@ set -eu export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' cd "<%= @expiry_dir %>" +rm -f z*.txt + osm2pgsql-replication update \ -d "<%= @source_database %>" \ --max-diff-size "<%= @diff_size %>" \ diff --git a/cookbooks/web/recipes/statistics.rb b/cookbooks/web/recipes/statistics.rb index 84848b003..88c85a6fe 100644 --- a/cookbooks/web/recipes/statistics.rb +++ b/cookbooks/web/recipes/statistics.rb @@ -19,7 +19,6 @@ include_recipe "web::base" -ruby = "ruby#{node[:ruby][:version]}" rails_directory = "#{node[:web][:base_directory]}/rails" template "/usr/local/bin/statistics" do @@ -27,7 +26,7 @@ template "/usr/local/bin/statistics" do owner "root" group "root" mode "755" - variables :ruby => ruby, :directory => rails_directory + variables :ruby => node[:ruby][:interpreter], :directory => rails_directory end systemd_service "web-statistics" do diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb index debc4db50..cb71e73a5 100644 --- a/cookbooks/web/resources/rails_port.rb +++ b/cookbooks/web/resources/rails_port.rb @@ -408,11 +408,25 @@ action :create do action :delete end + directory "#{rails_directory}/vendor/bundle/ruby/#{node[:ruby][:version]}.0" do + owner new_resource.user + group new_resource.group + mode "775" + recursive true + end + + bundle_config "#{rails_directory}" do + user new_resource.user + group new_resource.group + settings "deployment" => "true", + "build.nokogiri" => "--use-system-libraries" + end + bundle_install "#{rails_directory}" do action :nothing - user "root" - group "root" - environment "NOKOGIRI_USE_SYSTEM_LIBRARIES" => "yes" + user new_resource.user + group new_resource.group + subscribes :run, "directory[#{rails_directory}/vendor/bundle/ruby/#{node[:ruby][:version]}.0]" subscribes :run, "git[#{rails_directory}]" end @@ -439,7 +453,7 @@ action :create do only_if { new_resource.build_assets } end - bundle_exec "#{rails_directory}/app/assets/javascripts/i18n" do + bundle_exec "#{rails_directory}/config/i18n-js.yml" do action :nothing directory rails_directory command "rails i18n:js:export" @@ -449,7 +463,18 @@ action :create do user new_resource.user group new_resource.group subscribes :run, "git[#{rails_directory}]" - only_if { new_resource.build_assets } + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n-js.yml") } + end + + bundle_exec "#{rails_directory}/config/i18n.yml" do + action :nothing + directory rails_directory + command "i18n export" + environment "HOME" => rails_directory + user new_resource.user + group new_resource.group + subscribes :run, "git[#{rails_directory}]" + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n.yml") } end bundle_exec "#{rails_directory}/public/assets" do @@ -466,7 +491,8 @@ action :create do subscribes :run, "file[#{rails_directory}/config/settings.local.yml]" subscribes :run, "file[#{rails_directory}/config/storage.yml]" subscribes :run, "bundle_exec[#{rails_directory}/package.json]" - subscribes :run, "bundle_exec[#{rails_directory}/app/assets/javascripts/i18n]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n-js.yml]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n.yml]" only_if { new_resource.build_assets } end diff --git a/cookbooks/web/templates/default/apache.frontend.erb b/cookbooks/web/templates/default/apache.frontend.erb index 39f2a6007..f05b99b5c 100644 --- a/cookbooks/web/templates/default/apache.frontend.erb +++ b/cookbooks/web/templates/default/apache.frontend.erb @@ -2,7 +2,7 @@ # # Setup logging -# +# SetEnvIfNoCase Authorization "^Basic " AUTH_METHOD=basic SetEnvIfNoCase Authorization "^OAuth " AUTH_METHOD=oauth1 SetEnvIfNoCase Authorization "^Bearer " AUTH_METHOD=oauth2 @@ -108,6 +108,13 @@ ErrorLog /var/log/apache2/error.log RewriteCond "%{QUERY_STRING}" "^q=abcde&t=20" RewriteRule "^/api/0\.6/notes/search$" - [R=429,L] + # + # Ignore GoogleAssociationService request storm + # https://en.osm.town/@osm_tech/114205363076771822 + # + RewriteCond %{HTTP_USER_AGENT} "GoogleAssociationService" + RewriteRule "^/\.well-known/assetlinks\.json$" - [R=429,L] + # # Force special MIME type for crossdomain.xml files # @@ -186,6 +193,7 @@ ErrorLog /var/log/apache2/error.log RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] diff --git a/cookbooks/web/templates/default/api-statistics.erb b/cookbooks/web/templates/default/api-statistics.erb index ce3af0b34..b6425ec88 100644 --- a/cookbooks/web/templates/default/api-statistics.erb +++ b/cookbooks/web/templates/default/api-statistics.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "apache_log_regex" require "file-tail" diff --git a/cookbooks/wiki/attributes/default.rb b/cookbooks/wiki/attributes/default.rb index 0cde6a313..b25f9e481 100644 --- a/cookbooks/wiki/attributes/default.rb +++ b/cookbooks/wiki/attributes/default.rb @@ -1,2 +1,8 @@ -# Force apache to listen only on localhost -# default[:apache][:listen_address] = "127.0.0.1" +default[:wiki][:site_name] = "wiki.openstreetmap.org" +default[:wiki][:site_aliases] = [ + "wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", + "osm.wiki", "www.osm.wiki", "wiki.osm.wiki" +] +default[:wiki][:site_notice] = nil +default[:wiki][:site_readonly] = nil +default[:wiki][:mediawiki_version] = "1.39" diff --git a/cookbooks/wiki/recipes/default.rb b/cookbooks/wiki/recipes/default.rb index 74e96d2cc..4daec683c 100644 --- a/cookbooks/wiki/recipes/default.rb +++ b/cookbooks/wiki/recipes/default.rb @@ -19,6 +19,8 @@ include_recipe "mediawiki" +site_name = node[:wiki][:site_name] + passwords = data_bag_item("wiki", "passwords") package "lua5.1" # newer versions do not work with Scribuntu! @@ -27,9 +29,10 @@ apache_site "default" do action [:disable] end -mediawiki_site "wiki.openstreetmap.org" do - aliases ["wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", - "osm.wiki", "www.osm.wiki", "wiki.osm.wiki"] +mediawiki_site site_name do + aliases node[:wiki][:site_aliases] + + version node[:wiki][:mediawiki_version] fpm_max_children 200 fpm_start_servers 25 @@ -54,54 +57,67 @@ mediawiki_site "wiki.openstreetmap.org" do hcaptcha_public_key "b67a410b-955e-4049-b432-f9c00e0202c0" hcaptcha_private_key passwords["hcaptcha"] - # site_notice "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." - # site_readonly "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." + namespaces "DE" => { :id => 200, :talk_id => 201 }, + "FR" => { :id => 202, :talk_id => 203 }, + "ES" => { :id => 204, :talk_id => 205 }, + "IT" => { :id => 206, :talk_id => 207 }, + "NL" => { :id => 208, :talk_id => 209 }, + "RU" => { :id => 210, :talk_id => 211 }, + "JA" => { :id => 212, :talk_id => 213 }, + "Proposal" => { :id => 3000, :talk_id => 3001 } + + force_ui_messages %w[mainpage-url mapfeatures-url contributors-url helppage blogs-url shop-url sitesupport-url] + + watch_category_membership true + + site_notice node[:wiki][:site_notice] + site_readonly node[:wiki][:site_readonly] end mediawiki_extension "CodeEditor" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "CodeMirror" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "Scribunto" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Scribunto.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Wikibase" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Wikibase.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "OsmWikibase" do - site "wiki.openstreetmap.org" + site site_name repository "https://github.com/nyurik/OsmWikibase.git" reference "master" end mediawiki_extension "Echo" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Echo.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Thanks" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Thanks.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "TimedMediaHandler" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "MultiMaps" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-MultiMaps.inc.php.erb" template_cookbook "wiki" variables :thunderforest_key => passwords["thunderforest"] @@ -109,37 +125,37 @@ mediawiki_extension "MultiMaps" do end mediawiki_extension "JsonConfig" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-JsonConfig.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Kartographer" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Kartographer.inc.php.erb" template_cookbook "wiki" end -cookbook_file "/srv/wiki.openstreetmap.org/osm_logo_wiki.png" do +cookbook_file "/srv/#{site_name}/osm_logo_wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -template "/srv/wiki.openstreetmap.org/robots.txt" do +template "/srv/#{site_name}/robots.txt" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" source "robots.txt.erb" end -cookbook_file "/srv/wiki.openstreetmap.org/favicon.ico" do +cookbook_file "/srv/#{site_name}/favicon.ico" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -directory "/srv/wiki.openstreetmap.org/dump" do +directory "/srv/#{site_name}/dump" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "0775" @@ -149,13 +165,13 @@ systemd_service "wiki-dump" do description "Wiki dump" type "oneshot" exec_start "/usr/bin/php w/maintenance/dumpBackup.php --full --quiet --output=gzip:dump/dump.xml.gz" - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" nice 19 sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-dump" do @@ -173,14 +189,14 @@ systemd_service "wiki-rdf-dump" do exec_start [ "/usr/bin/php w/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki wiki --format ttl --flavor full-dump --entity-type item --entity-type property --no-cache --output /tmp/wikibase-rdf.ttl", "/bin/gzip -9 /tmp/wikibase-rdf.ttl", - "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/wiki.openstreetmap.org/dump/wikibase-rdf.ttl.gz" + "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/#{site_name}/dump/wikibase-rdf.ttl.gz" ] - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-rdf-dump" do diff --git a/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb b/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb index 6f7e8d6eb..d609a3d57 100644 --- a/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb +++ b/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb @@ -89,7 +89,7 @@ $wgWBClientSettings['showExternalRecentChanges'] = true; // Base URL for building links to the repository. // Assumes your wiki is setup as "http://repo.example.org/wiki/" // This can be protocol relative, such as "//www.wikidata.org" -$wgWBClientSettings['repoUrl'] = "https://wiki.openstreetmap.org"; +$wgWBClientSettings['repoUrl'] = "https://<%= @site %>"; // This setting is optional if you have the same type of setup for your // repo and client. It will default to using the client's $wgArticlePath setting, diff --git a/roles/base.rb b/roles/base.rb index 2a4f732a5..ee9b003c6 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -11,10 +11,6 @@ default_attributes( } }, :networking => { - :roles => { - :internal => { :metric => 200 }, - :external => { :metric => 100 } - }, :search => ["openstreetmap.org"] }, :prometheus => { diff --git a/roles/blogs.rb b/roles/blogs.rb index 3881d1396..87c71902a 100644 --- a/roles/blogs.rb +++ b/roles/blogs.rb @@ -1,6 +1,12 @@ name "blogs" description "Role applied to all blog aggregators" +default_attributes( + :ruby => { + :version => "3.3" + } +) + run_list( "recipe[blogs]" ) diff --git a/roles/bytemark.rb b/roles/bytemark.rb index 8f23bd5d0..109a17208 100644 --- a/roles/bytemark.rb +++ b/roles/bytemark.rb @@ -3,30 +3,7 @@ description "Role applied to all servers at Bytemark" default_attributes( :hosted_by => "Bytemark", - :location => "York, England", - :networking => { - :roles => { - :internal => { - :inet => { - :prefix => "20", - :gateway => "10.0.32.20", - :routes => { - "10.0.0.0/8" => { :via => "10.0.32.20" } - } - } - }, - :external => { - :inet => { - :prefix => "28", - :gateway => "89.16.162.17" - }, - :inet6 => { - :prefix => "64", - :gateway => "fe80::1" - } - } - } - } + :location => "York, England" ) override_attributes( diff --git a/roles/culebre.rb b/roles/culebre.rb index d798bddae..8f266566a 100644 --- a/roles/culebre.rb +++ b/roles/culebre.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.9" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp68s0f0 enp68s0f1 enp68s0f2 enp68s0f3] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.105", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.105" }, :inet6 => { - :address => "2001:470:1:b3b::9", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::9" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.105", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.105" }, :inet6 => { - :address => "2001:4d78:fe03:1c::9", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::9" } } } @@ -83,6 +63,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[tile]" ) diff --git a/roles/dev.rb b/roles/dev.rb index f85849e88..432fe0d1c 100644 --- a/roles/dev.rb +++ b/roles/dev.rb @@ -116,7 +116,7 @@ default_attributes( :revision => "master", :cgimap_repository => "https://github.com/zerebubuth/openstreetmap-cgimap.git", :cgimap_revision => "master", - :aliases => ["api06.dev.openstreetmap.org"] + :aliases => ["master.apis.dev.osm.org", "api06.dev.openstreetmap.org", "api06.dev.osm.org"] }, :tomh => { :repository => "https://github.com/tomhughes/openstreetmap-website.git", @@ -159,6 +159,9 @@ default_attributes( } } }, + :ruby => { + :fullstaq => true + }, :sysctl => { :postgres => { :comment => "Increase shared memory for postgres", diff --git a/roles/dribble.rb b/roles/dribble.rb index 33707cfa4..b0bba8021 100644 --- a/roles/dribble.rb +++ b/roles/dribble.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.4" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.132", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.132" }, :inet6 => { - :address => "2001:470:1:fa1::4", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::4" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.100", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.100" }, :inet6 => { - :address => "2001:4d78:500:5e3::4", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::4" } } } @@ -60,6 +40,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[vectortile]" ) diff --git a/roles/dulcy.rb b/roles/dulcy.rb index d9d8fb79f..cc6b24fe6 100644 --- a/roles/dulcy.rb +++ b/roles/dulcy.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.9" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp1s0f0 enp1s0f1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.137", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.137" }, :inet6 => { - :address => "2001:470:1:fa1::9", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::9" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.105", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.105" }, :inet6 => { - :address => "2001:4d78:500:5e3::9", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::9" } } } @@ -62,13 +42,13 @@ default_attributes( :nominatim => { :dbcluster => "17/main", :flatnode_file => "/srv/nominatim.openstreetmap.org/planet-project/nodes.store", - :enable_qa_tiles => true, + :enable_qa_tiles => false, :api_workers => 14, :api_pool_size => 10 } ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[nominatim]" ) diff --git a/roles/eddie.rb b/roles/eddie.rb index 61e452455..2acf524e1 100644 --- a/roles/eddie.rb +++ b/roles/eddie.rb @@ -6,7 +6,6 @@ default_attributes( :interfaces => { :internal => { :interface => "enp1s0f0.2801", - :role => :internal, :inet => { :address => "10.0.0.10" } diff --git a/roles/equinix-ams-public.rb b/roles/equinix-ams-public.rb new file mode 100644 index 000000000..b89ae21bb --- /dev/null +++ b/roles/equinix-ams-public.rb @@ -0,0 +1,44 @@ +name "equinix-ams-public" +description "Role applied to all public servers at Equinix Amsterdam" + +default_attributes( + :networking => { + :interfaces => { + :henet => { + :interface => "bond0.3", + :role => :external, + :metric => 150, + :source_route_table => 3, + :inet => { + :prefix => "27", + :gateway => "184.104.179.129" + }, + :inet6 => { + :prefix => "64", + :gateway => "2001:470:1:fa1::1", + :routes => { + "2600:9000::/28" => { :table => 3, :type => "unreachable" } + } + } + }, + :equinix => { + :interface => "bond0.103", + :role => :external, + :metric => 100, + :source_route_table => 103, + :inet => { + :prefix => "27", + :gateway => "82.199.86.97" + }, + :inet6 => { + :prefix => "64", + :gateway => "2001:4d78:500:5e3::1" + } + } + } + } +) + +run_list( + "role[equinix-ams]" +) diff --git a/roles/equinix-ams.rb b/roles/equinix-ams.rb index ac8c6a885..c63522bbc 100644 --- a/roles/equinix-ams.rb +++ b/roles/equinix-ams.rb @@ -3,18 +3,23 @@ description "Role applied to all servers at Equinix Amsterdam" default_attributes( :networking => { - :roles => { + :interfaces => { :internal => { + :interface => "bond0", + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.48.14", :routes => { "10.0.0.0/8" => { :via => "10.0.48.14" } } + }, + :bond => { + :mode => "802.3ad", + :lacprate => "fast", + :xmithashpolicy => "layer3+4" } - }, - :external => { - :zone => "ams" } } }, diff --git a/roles/equinix-dub-public.rb b/roles/equinix-dub-public.rb new file mode 100644 index 000000000..343dfb4f9 --- /dev/null +++ b/roles/equinix-dub-public.rb @@ -0,0 +1,44 @@ +name "equinix-dub-public" +description "Role applied to all public servers at Equinix Dublin" + +default_attributes( + :networking => { + :interfaces => { + :henet => { + :interface => "bond0.101", + :role => :external, + :metric => 150, + :source_route_table => 101, + :inet => { + :prefix => "27", + :gateway => "184.104.226.97" + }, + :inet6 => { + :prefix => 64, + :gateway => "2001:470:1:b3b::1", + :routes => { + "2600:9000::/28" => { :table => 101, :type => "unreachable" } + } + } + }, + :equinix => { + :interface => "bond0.203", + :role => :external, + :metric => 100, + :source_route_table => 203, + :inet => { + :prefix => "27", + :gateway => "87.252.214.97" + }, + :inet6 => { + :prefix => 64, + :gateway => "2001:4d78:fe03:1c::1" + } + } + } + } +) + +run_list( + "role[equinix-dub]" +) diff --git a/roles/equinix-dub.rb b/roles/equinix-dub.rb index 9652c2c8c..741c09ab7 100644 --- a/roles/equinix-dub.rb +++ b/roles/equinix-dub.rb @@ -12,29 +12,27 @@ default_attributes( } }, :networking => { - :roles => { + :interfaces => { :internal => { + :interface => "bond0", + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.64.2", :routes => { "10.0.0.0/8" => { :via => "10.0.64.2" } - } - } - }, - :external => { - :zone => "dub", - :inet => { + }, :rules => [ { :to => "10.0.0.0/8", :table => "main", :priority => 50 }, { :to => "172.16.0.0/12", :table => "main", :priority => 50 }, { :to => "192.168.0.0/16", :table => "main", :priority => 50 } ] }, - :inet6 => { - :rules => [ - { :to => "2600:9000::/28", :table => 150, :priority => 100 } - ] + :bond => { + :mode => "802.3ad", + :lacprate => "fast", + :xmithashpolicy => "layer3+4" } } } diff --git a/roles/faffy.rb b/roles/faffy.rb index 34b480823..e02510d18 100644 --- a/roles/faffy.rb +++ b/roles/faffy.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.3" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.131", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.131" }, :inet6 => { - :address => "2001:470:1:fa1::3", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::3" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.99", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.99" }, :inet6 => { - :address => "2001:4d78:500:5e3::3", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::3" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[dev]" ) diff --git a/roles/fafnir.rb b/roles/fafnir.rb index 7ca7094bc..e7edc263c 100644 --- a/roles/fafnir.rb +++ b/roles/fafnir.rb @@ -19,47 +19,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.2" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.98", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.98" }, :inet6 => { - :address => "2001:470:1:b3b::2", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::2" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.98", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.98" }, :inet6 => { - :address => "2001:4d78:fe03:1c::2", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::2" } } } @@ -82,7 +62,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[gateway]", "role[mail]", diff --git a/roles/firefishynet.rb b/roles/firefishynet.rb index 150f62d6f..bb4224f70 100644 --- a/roles/firefishynet.rb +++ b/roles/firefishynet.rb @@ -3,7 +3,7 @@ description "Role applied to all servers at Firefishy" default_attributes( :networking => { - :roles => { + :interfaces => { :internal => { :inet => { :prefix => "24", diff --git a/roles/fume.rb b/roles/fume.rb index 5cc06a4b1..af951e1f0 100644 --- a/roles/fume.rb +++ b/roles/fume.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.16" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.112", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.112" }, :inet6 => { - :address => "2001:470:1:b3b::10", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::10" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.112", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.112" }, :inet6 => { - :address => "2001:4d78:fe03:1c::10", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::10" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[community]" ) diff --git a/roles/geodns.rb b/roles/geodns.rb index a2ad092f3..a1df01532 100644 --- a/roles/geodns.rb +++ b/roles/geodns.rb @@ -12,8 +12,10 @@ default_attributes( :list => false, :transfer_logging => false, :hosts_allow => [ - "184.104.226.102", # idris - "2001:470:1:b3b::6" # idris + "184.104.226.102", # idris HE + "2001:470:1:b3b::6", # idris HE + "87.252.214.102", # idris Equinix + "2001:4d78:fe03:1c::6" # idris Equinix ] } } diff --git a/roles/grisu.rb b/roles/grisu.rb index 525e028d2..e7fd2f899 100644 --- a/roles/grisu.rb +++ b/roles/grisu.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.17" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.113", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.113" }, :inet6 => { - :address => "2001:470:1:b3b::11", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::11" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.113", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.113" }, :inet6 => { - :address => "2001:4d78:fe03:1c::11", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::11" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[overpass-query]" ) diff --git a/roles/horntail.rb b/roles/horntail.rb index d071856f4..b09bf796b 100644 --- a/roles/horntail.rb +++ b/roles/horntail.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.10" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0 enp25s0f1] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.106", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.106" }, :inet6 => { - :address => "2001:470:1:b3b::a", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::a" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.106", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.106" }, :inet6 => { - :address => "2001:4d78:fe03:1c::a", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::a" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[geodns]", "role[planet]" ) diff --git a/roles/idris.rb b/roles/idris.rb index 7467f67af..2a91aaf73 100644 --- a/roles/idris.rb +++ b/roles/idris.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.6" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.102", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.102" }, :inet6 => { - :address => "2001:470:1:b3b::6", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::6" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.102", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.102" }, :inet6 => { - :address => "2001:4d78:fe03:1c::6", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::6" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[chef-server]", "role[chef-repository]", diff --git a/roles/imagery.rb b/roles/imagery.rb index d6caa11e9..4a4a033a3 100644 --- a/roles/imagery.rb +++ b/roles/imagery.rb @@ -20,21 +20,13 @@ default_attributes( "net.core.somaxconn" => 10000 } } - }, - :nginx => { - :cache => { - :fastcgi => { - :enable => true, - :keys_zone => "fastcgi_cache_zone:256M", - :inactive => "45d", - :max_size => "51200M" - } - } } ) run_list( "recipe[imagery::au_agri]", + "recipe[imagery::au_act_aerial]", + "recipe[imagery::au_vic_melbourne_aerial]", "recipe[imagery::gb_ea]", "recipe[imagery::gb_hampshire_aerial]", "recipe[imagery::gb_os_sv]", diff --git a/roles/karm.rb b/roles/karm.rb index ac28c48d4..bb736d72c 100644 --- a/roles/karm.rb +++ b/roles/karm.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.50" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp1s0f0 enp1s0f1 enp2s0f0 enp2s0f1] } } diff --git a/roles/konqi.rb b/roles/konqi.rb index 4b1fbcea6..2ef8ebd84 100644 --- a/roles/konqi.rb +++ b/roles/konqi.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.7" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.103", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.103" }, :inet6 => { - :address => "2001:470:1:b3b::7", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::7" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.103", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.103" }, :inet6 => { - :address => "2001:4d78:fe03:1c::7", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::7" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[wiki]" ) diff --git a/roles/lockheed.rb b/roles/lockheed.rb index 09eddcc8b..82acb48b4 100644 --- a/roles/lockheed.rb +++ b/roles/lockheed.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.16" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno49 eno50] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.144", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.144" }, :inet6 => { - :address => "2001:470:1:fa1::10", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::10" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.112", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.112" }, :inet6 => { - :address => "2001:4d78:500:5e3::10", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::10" } } } @@ -85,18 +65,24 @@ default_attributes( }, :nginx => { :cache => { + :fastcgi => { + :enable => true, + :keys_zone => "fastcgi_cache_zone:2048M", + :inactive => "180d", + :max_size => "196608M" + }, :proxy => { :enable => true, - :keys_zone => "proxy_cache_zone:256M", + :keys_zone => "proxy_cache_zone:2048M", :inactive => "180d", - :max_size => "51200M" + :max_size => "196608M" } } } ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[imagery]" ) diff --git a/roles/longma.rb b/roles/longma.rb index b8bf00af1..b64af9aa4 100644 --- a/roles/longma.rb +++ b/roles/longma.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.13" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp68s0f0np0 enp68s0f1np1 enp68s0f2np2 enp68s0f3np3] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.109", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.109" }, :inet6 => { - :address => "2001:470:1:b3b::d", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::d" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.109", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.109" }, :inet6 => { - :address => "2001:4d78:fe03:1c::d", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::d" } } } @@ -70,6 +50,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[nominatim]" ) diff --git a/roles/muirdris.rb b/roles/muirdris.rb index b77e4f8df..1adfbdc93 100644 --- a/roles/muirdris.rb +++ b/roles/muirdris.rb @@ -2,60 +2,50 @@ name "muirdris" description "Master role applied to muirdris" default_attributes( - :memcached => { - :memory_limit => 128 * 1024 - }, :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.15" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.111", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.111" }, :inet6 => { - :address => "2001:470:1:b3b::f", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::f" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.111", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.111" }, :inet6 => { - :address => "2001:4d78:fe03:1c::f", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::f" } } } + }, + :wiki => { + :site_name => "test.wiki.openstreetmap.org", + :site_aliases => [], + :site_notice => "TEST INSTANCE: Use wiki.openstreetmap.org for real work", + :mediawiki_version => "1.43" + } +) + +override_attributes( + :memcached => { + :memory_limit => 128 * 1024 } ) run_list( - "role[equinix-dub]", - "role[gps-tile]" + "role[equinix-dub-public]", + "role[gps-tile]", + "role[wiki]" ) diff --git a/roles/naga.rb b/roles/naga.rb index 5bb605112..1bcf3c176 100644 --- a/roles/naga.rb +++ b/roles/naga.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.8" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.104", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.104" }, :inet6 => { - :address => "2001:470:1:b3b::8", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::8" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.104", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.104" }, :inet6 => { - :address => "2001:4d78:fe03:1c::8", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::8" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[subversion]", "role[trac]", diff --git a/roles/norbert.rb b/roles/norbert.rb index ef3d33dbd..357dc0d5a 100644 --- a/roles/norbert.rb +++ b/roles/norbert.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.17" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0 enp25s0f1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.145", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.145" }, :inet6 => { - :address => "2001:470:1:fa1::11", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::11" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.113", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.113" }, :inet6 => { - :address => "2001:4d78:500:5e3::11", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::11" } } } @@ -56,7 +36,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[geodns]", "role[backup]", "role[planet]", diff --git a/roles/odin.rb b/roles/odin.rb index 1544fb9b3..9d5bd1cfc 100644 --- a/roles/odin.rb +++ b/roles/odin.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.15" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1np0 eno2np1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.143", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.143" }, :inet6 => { - :address => "2001:470:1:fa1::f", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::f" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.111", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.111" }, :inet6 => { - :address => "2001:4d78:500:5e3::f", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::f" } } } @@ -145,6 +125,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[tile]" ) diff --git a/roles/osuosl.rb b/roles/osuosl.rb index 99a3a992f..83fd46a72 100644 --- a/roles/osuosl.rb +++ b/roles/osuosl.rb @@ -11,8 +11,10 @@ default_attributes( :location => "Corvallis, Oregon", :timezone => "US/Pacific", :networking => { - :roles => { + :interfaces => { :external => { + :interface => "bond0", + :role => :external, :inet => { :prefix => "28", :gateway => "140.211.167.97" diff --git a/roles/piasa.rb b/roles/piasa.rb index b1e775ad4..085fdc66e 100644 --- a/roles/piasa.rb +++ b/roles/piasa.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.101" }, diff --git a/roles/ridley.rb b/roles/ridley.rb index 0e4a66522..2a6b166c1 100644 --- a/roles/ridley.rb +++ b/roles/ridley.rb @@ -20,14 +20,12 @@ default_attributes( :interfaces => { :external => { :interface => "eth0.2800", - :role => :external, :inet => { :address => "193.60.236.19" } }, :internal => { :interface => "eth0.2801", - :role => :internal, :inet => { :address => "10.0.0.3" } @@ -37,7 +35,7 @@ default_attributes( ) run_list( - "role[ucl]", + "role[ucl-public]", "role[hp-dl360-g6]", "role[gateway]", "role[foundation]", diff --git a/roles/shenron.rb b/roles/shenron.rb index f91677226..7f9f36a1c 100644 --- a/roles/shenron.rb +++ b/roles/shenron.rb @@ -19,14 +19,6 @@ default_attributes( "it87" ] }, - :prometheus => { - :metrics => { - :exim_queue_limit => { :metric => 250 } - } - } -) - -override_attributes( :networking => { :dnssec => "false", :interfaces => { @@ -45,6 +37,11 @@ override_attributes( } } } + }, + :prometheus => { + :metrics => { + :exim_queue_limit => { :metric => 250 } + } } ) diff --git a/roles/smaug.rb b/roles/smaug.rb index 4156fcb99..7d42c4f6e 100644 --- a/roles/smaug.rb +++ b/roles/smaug.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.14" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.110", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.110" }, :inet6 => { - :address => "2001:470:1:b3b::e", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::e" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.110", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.110" }, :inet6 => { - :address => "2001:4d78:fe03:1c::e", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::e" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[matomo]" ) diff --git a/roles/snap-01.rb b/roles/snap-01.rb index d7fb452a5..e741e2d51 100644 --- a/roles/snap-01.rb +++ b/roles/snap-01.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.49" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4] } } diff --git a/roles/snap-02.rb b/roles/snap-02.rb index 80a2b3598..2a63385b3 100644 --- a/roles/snap-02.rb +++ b/roles/snap-02.rb @@ -6,7 +6,6 @@ default_attributes( :interfaces => { :internal => { :interface => "eno1.2801", - :role => :internal, :inet => { :address => "10.0.0.4" } diff --git a/roles/snap-03.rb b/roles/snap-03.rb index 4956bdab7..0164f8563 100644 --- a/roles/snap-03.rb +++ b/roles/snap-03.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.50" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0np0 enp25s0f1np1] } } diff --git a/roles/spike-01.rb b/roles/spike-01.rb index 80b56c5e1..51ff42c63 100644 --- a/roles/spike-01.rb +++ b/roles/spike-01.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.3" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.99", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.99" }, :inet6 => { - :address => "2001:470:1:b3b::3", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::3" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.99", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.99" }, :inet6 => { - :address => "2001:4d78:fe03:1c::3", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::3" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-02.rb b/roles/spike-02.rb index 065be3faf..4e178bdec 100644 --- a/roles/spike-02.rb +++ b/roles/spike-02.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.4" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.100", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.100" }, :inet6 => { - :address => "2001:470:1:b3b::4", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::4" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.100", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.100" }, :inet6 => { - :address => "2001:4d78:fe03:1c::4", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::4" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-03.rb b/roles/spike-03.rb index b620aecfc..a24512eaa 100644 --- a/roles/spike-03.rb +++ b/roles/spike-03.rb @@ -5,47 +5,28 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", :role => :internal, :inet => { :address => "10.0.64.5" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.101", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.101" }, :inet6 => { - :address => "2001:470:1:b3b::5", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::5" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.101", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.101" }, :inet6 => { - :address => "2001:4d78:fe03:1c::5", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::5" } } } @@ -53,7 +34,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-06.rb b/roles/spike-06.rb index dd754a7ba..b5d84e6a6 100644 --- a/roles/spike-06.rb +++ b/roles/spike-06.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.6" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.134", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.134" }, :inet6 => { - :address => "2001:470:1:fa1::6", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::6" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.102", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.102" }, :inet6 => { - :address => "2001:4d78:500:5e3::6", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::6" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]", "role[web-statistics]", diff --git a/roles/spike-07.rb b/roles/spike-07.rb index f9d0d8f12..166432919 100644 --- a/roles/spike-07.rb +++ b/roles/spike-07.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.7" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.135", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.135" }, :inet6 => { - :address => "2001:470:1:fa1::7", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::7" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.103", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.103" }, :inet6 => { - :address => "2001:4d78:500:5e3::7", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::7" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-08.rb b/roles/spike-08.rb index c46260340..46f0be788 100644 --- a/roles/spike-08.rb +++ b/roles/spike-08.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.8" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.136", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.136" }, :inet6 => { - :address => "2001:470:1:fa1::8", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::8" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.104", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.104" }, :inet6 => { - :address => "2001:4d78:500:5e3::8", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::8" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/stormfly-03.rb b/roles/stormfly-03.rb index d81b06c9f..374d2ab90 100644 --- a/roles/stormfly-03.rb +++ b/roles/stormfly-03.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.99" }, diff --git a/roles/stormfly-04.rb b/roles/stormfly-04.rb index 3a7d1a7c3..bcc11518a 100644 --- a/roles/stormfly-04.rb +++ b/roles/stormfly-04.rb @@ -8,8 +8,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.100" }, diff --git a/roles/tabaluga.rb b/roles/tabaluga.rb index af8492b83..1ad1ee227 100644 --- a/roles/tabaluga.rb +++ b/roles/tabaluga.rb @@ -9,47 +9,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.14" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.142", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.142" }, :inet6 => { - :address => "2001:470:1:fa1::e", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::e" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.110", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.110" }, :inet6 => { - :address => "2001:4d78:500:5e3::e", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::e" } } } @@ -72,7 +52,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[blog-staging]", "role[taginfo]", diff --git a/roles/taginfo.rb b/roles/taginfo.rb index 40bffdce2..9b8227957 100644 --- a/roles/taginfo.rb +++ b/roles/taginfo.rb @@ -26,6 +26,9 @@ default_attributes( :passenger => { :max_pool_size => 50 }, + :ruby => { + :version => "3.3" + }, :taginfo => { :sites => [ { diff --git a/roles/tile.rb b/roles/tile.rb index 20bded97f..550c99f66 100644 --- a/roles/tile.rb +++ b/roles/tile.rb @@ -46,6 +46,9 @@ default_attributes( } } }, + :ruby => { + :fullstaq => false + }, :ssl => { :ct_report_uri => false }, diff --git a/roles/ucl-public.rb b/roles/ucl-public.rb new file mode 100644 index 000000000..7b7c6188a --- /dev/null +++ b/roles/ucl-public.rb @@ -0,0 +1,21 @@ +name "ucl-public" +description "Role applied to all public servers at UCL" + +default_attributes( + :networking => { + :interfaces => { + :external => { + :role => :external, + :metric => 100, + :inet => { + :prefix => "24", + :gateway => "193.60.236.254" + } + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/ucl.rb b/roles/ucl.rb index 516c925d3..1734989bf 100644 --- a/roles/ucl.rb +++ b/roles/ucl.rb @@ -4,8 +4,10 @@ description "Role applied to all servers at UCL" default_attributes( :location => "Slough, England", :networking => { - :roles => { + :interfaces => { :internal => { + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.0.3", @@ -13,13 +15,6 @@ default_attributes( "10.0.0.0/8" => { :via => "10.0.0.3" } } } - }, - :external => { - :zone => "ucl", - :inet => { - :prefix => "24", - :gateway => "193.60.236.254" - } } }, :wireguard => { diff --git a/roles/vhagar.rb b/roles/vhagar.rb index b5776ab2a..c8cd4a92f 100644 --- a/roles/vhagar.rb +++ b/roles/vhagar.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.5" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.133", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.133" }, :inet6 => { - :address => "2001:470:1:fa1::5", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::5" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.101", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.101" }, :inet6 => { - :address => "2001:4d78:500:5e3::5", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::5" } } } @@ -60,6 +40,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[nominatim]" ) diff --git a/roles/web-frontend.rb b/roles/web-frontend.rb index 396cf36db..e7c0e5850 100644 --- a/roles/web-frontend.rb +++ b/roles/web-frontend.rb @@ -30,6 +30,9 @@ default_attributes( :passenger => { :max_pool_size => 50 }, + :ruby => { + :fullstaq => true + }, :exim => { :local_domains => ["messages.openstreetmap.org"], :routes => { diff --git a/roles/wiki.rb b/roles/wiki.rb index 19a31e3db..652726346 100644 --- a/roles/wiki.rb +++ b/roles/wiki.rb @@ -59,7 +59,7 @@ default_attributes( :mysqld => { :innodb_buffer_pool_size => "4G", :key_buffer_size => "64M", - :max_connections => "500", + :max_connections => "800", :sort_buffer_size => "8M", :tmp_table_size => "128M" } diff --git a/roles/ysera.rb b/roles/ysera.rb index bf256d44c..8d3c16f87 100644 --- a/roles/ysera.rb +++ b/roles/ysera.rb @@ -6,14 +6,12 @@ default_attributes( :interfaces => { :internal => { :interface => "eno1np0.2801", - :role => :internal, :inet => { :address => "10.0.0.15" } }, :external => { :interface => "eno1np0.2800", - :role => :external, :inet => { :address => "193.60.236.22" } @@ -115,6 +113,6 @@ default_attributes( ) run_list( - "role[ucl]", + "role[ucl-public]", "role[tile]" ) diff --git a/test/data_bags/accounts/serverinfo.json b/test/data_bags/accounts/serverinfo.json new file mode 100644 index 000000000..2b01af603 --- /dev/null +++ b/test/data_bags/accounts/serverinfo.json @@ -0,0 +1,6 @@ +{ + "id": "serverinfo", + "uid": "534", + "comment": "hardware.openstreetmap.org", + "manage_home": false +}