From: Tom Hughes Date: Mon, 23 Jan 2023 22:40:19 +0000 (+0000) Subject: Merge remote-tracking branch 'github/pull/565' X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/3ec71f5485e7f07c4ff2306d65d780951b61d71e?hp=8055288bcfd56e518bd3500158945cefdf804b11 Merge remote-tracking branch 'github/pull/565' --- diff --git a/cookbooks/db/attributes/default.rb b/cookbooks/db/attributes/default.rb index 4602a00c3..56e96007e 100644 --- a/cookbooks/db/attributes/default.rb +++ b/cookbooks/db/attributes/default.rb @@ -1,3 +1,3 @@ -default[:db][:cluster] = "9.5/main" +default[:db][:cluster] = "15/main" -default[:postgresql][:versions] |= ["9.5", "15"] +default[:postgresql][:versions] |= ["15"] diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index a9e85008e..b8c589796 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -270,9 +270,9 @@ node[:postgresql][:versions].each do |version| package "postgresql-#{version}-postgis-3" end -if node[:postgresql][:clusters][:"14/main"] +if node[:postgresql][:clusters][:"15/main"] postgresql_user "apis" do - cluster "14/main" + cluster "15/main" end template "/usr/local/bin/cleanup-rails-assets" do @@ -331,12 +331,12 @@ if node[:postgresql][:clusters][:"14/main"] secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do - cluster "14/main" + cluster "15/main" owner "apis" end postgresql_extension "#{database_name}_btree_gist" do - cluster "14/main" + cluster "15/main" database database_name extension "btree_gist" end @@ -377,7 +377,7 @@ if node[:postgresql][:clusters][:"14/main"] group "apis" repository details[:repository] revision details[:revision] - database_port node[:postgresql][:clusters][:"14/main"][:port] + database_port node[:postgresql][:clusters][:"15/main"][:port] database_name database_name database_username "apis" email_from "OpenStreetMap " @@ -448,7 +448,7 @@ if node[:postgresql][:clusters][:"14/main"] group "root" mode "640" variables :cgimap_port => cgimap_port, - :database_port => node[:postgresql][:clusters][:"14/main"][:port], + :database_port => node[:postgresql][:clusters][:"15/main"][:port], :database_name => database_name, :log_directory => log_directory end @@ -518,7 +518,7 @@ if node[:postgresql][:clusters][:"14/main"] postgresql_database database_name do action :drop - cluster "14/main" + cluster "15/main" end end end diff --git a/cookbooks/postgresql/recipes/default.rb b/cookbooks/postgresql/recipes/default.rb index a57e1b348..affe9f502 100644 --- a/cookbooks/postgresql/recipes/default.rb +++ b/cookbooks/postgresql/recipes/default.rb @@ -33,12 +33,25 @@ node[:postgresql][:versions].each do |version| defaults = node[:postgresql][:settings][:defaults] || {} settings = node[:postgresql][:settings][version] || {} + standby_mode = settings[:standby_mode] || defaults[:standby_mode] + primary_conninfo = settings[:primary_conninfo] || defaults[:primary_conninfo] + restore_command = settings[:restore_command] || defaults[:restore_command] + + passwords = if primary_conninfo + data_bag_item(primary_conninfo[:passwords][:bag], + primary_conninfo[:passwords][:item]) + end + template "/etc/postgresql/#{version}/main/postgresql.conf" do source "postgresql.conf.erb" owner "postgres" group "postgres" mode "644" - variables :version => version, :defaults => defaults, :settings => settings + variables :version => version, + :defaults => defaults, + :settings => settings, + :primary_conninfo => primary_conninfo, + :passwords => passwords notifies :reload, "service[postgresql]" only_if { ::Dir.exist?("/etc/postgresql/#{version}/main") } end @@ -74,16 +87,7 @@ node[:postgresql][:versions].each do |version| only_if { ::Dir.exist?("/var/lib/postgresql/#{version}/main") } end - standby_mode = settings[:standby_mode] || defaults[:standby_mode] - primary_conninfo = settings[:primary_conninfo] || defaults[:primary_conninfo] - restore_command = settings[:restore_command] || defaults[:restore_command] - - if restore_command || standby_mode == "on" - passwords = if primary_conninfo - data_bag_item(primary_conninfo[:passwords][:bag], - primary_conninfo[:passwords][:item]) - end - + if version.to_f < 12 && (restore_command || standby_mode == "on") template "/var/lib/postgresql/#{version}/main/recovery.conf" do source "recovery.conf.erb" owner "postgres" @@ -103,6 +107,18 @@ node[:postgresql][:versions].each do |version| only_if { ::Dir.exist?("/var/lib/postgresql/#{version}/main") } end end + + if version.to_f > 11 && standby_mode == "on" + file "/var/lib/postgresql/#{version}/main/standby.signal" do + owner "postgres" + group "postgres" + mode "640" + end + else + file "/var/lib/postgresql/#{version}/main/standby.signal" do + action :delete + end + end end service "postgresql" do @@ -182,5 +198,6 @@ prometheus_exporter "postgres" do "PG_EXPORTER_AUTO_DISCOVER_DATABASES" => "true", "PG_EXPORTER_EXCLUDE_DATABASES" => "postgres,template0,template1" restrict_address_families "AF_UNIX" + remove_ipc false subscribes :restart, "template[/etc/prometheus/exporters/postgres_queries.yml]" end diff --git a/cookbooks/postgresql/templates/default/postgresql.conf.erb b/cookbooks/postgresql/templates/default/postgresql.conf.erb index d75c2f427..55053f5f8 100644 --- a/cookbooks/postgresql/templates/default/postgresql.conf.erb +++ b/cookbooks/postgresql/templates/default/postgresql.conf.erb @@ -86,6 +86,14 @@ archive_mode = <%= @settings[:archive_mode] || @defaults[:archive_mode] %> archive_command = '<%= @settings[:archive_command] || @defaults[:archive_command] %>' <% end -%> +# - Archive Recovery - + +# These are only used in recovery mode. + +<% if @settings[:restore_command] || @defaults[:restore_command] -%> +restore_command = '<%= @settings[:restore_command] || @defaults[:restore_command] %>' +<% end -%> + #------------------------------------------------------------------------------ # REPLICATION #------------------------------------------------------------------------------ @@ -99,6 +107,9 @@ max_replication_slots = <%= @settings[:max_replication_slots] || @defaults[:max_ # - Standby Servers - +<% if @primary_conninfo -%> +primary_conninfo = 'host=<%= @primary_conninfo[:host] %> port=<%= @primary_conninfo[:port] %> user=<%= @primary_conninfo[:user] %> password=<%= @passwords[@primary_conninfo[:user]] %>' +<% end -%> hot_standby = <%= @settings[:hot_standby] || @defaults[:hot_standby] %> hot_standby_feedback = <%= @settings[:hot_standby_feedback] || @defaults[:hot_standby_feedback] %> @@ -138,7 +149,9 @@ log_line_prefix = '%t ' # - Query/Index Statistics Collector - track_activity_query_size = <%= @settings[:track_activity_query_size] || @defaults[:track_activity_query_size] %> +<% if @version.to_f < 15 -%> stats_temp_directory = '/run/postgresql/<%= @version %>-main.pg_stat_tmp' +<% end -%> #------------------------------------------------------------------------------ # AUTOVACUUM PARAMETERS diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb index f34af60da..845312f2e 100644 --- a/cookbooks/prometheus/recipes/server.rb +++ b/cookbooks/prometheus/recipes/server.rb @@ -111,7 +111,8 @@ archive_file "#{cache_dir}/karma-linux-amd64.tar.gz" do subscribes :extract, "remote_file[#{cache_dir}/karma-linux-amd64.tar.gz]" end -promscale_version = "0.16.0" +promscale_version = "0.17.0" +promscale_extension_version = "0.8.0-1" database_version = node[:timescaledb][:database_version] database_cluster = "#{database_version}/main" @@ -122,6 +123,15 @@ package %W[ promscale-extension-postgresql-#{database_version} ] +package "promscale-extension-postgresql-#{database_version}" do + version promscale_extension_version +end + +apt_preference "promscale-extension-postgresql" do + pin "version #{promscale_extension_version}" + pin_priority "1100" +end + postgresql_user "prometheus" do cluster database_cluster superuser true diff --git a/cookbooks/prometheus/resources/exporter.rb b/cookbooks/prometheus/resources/exporter.rb index 3087f9c93..581c961dd 100644 --- a/cookbooks/prometheus/resources/exporter.rb +++ b/cookbooks/prometheus/resources/exporter.rb @@ -36,6 +36,7 @@ property :proc_subset, String property :private_devices, [true, false] property :protect_clock, [true, false] property :restrict_address_families, [String, Array] +property :remove_ipc, [true, false] property :system_call_filter, [String, Array] property :service, :kind_of => String property :scrape_interval, :kind_of => String @@ -60,6 +61,7 @@ action :create do private_devices new_resource.private_devices if new_resource.property_is_set?(:private_devices) protect_clock new_resource.protect_clock if new_resource.property_is_set?(:protect_clock) restrict_address_families new_resource.restrict_address_families if new_resource.property_is_set?(:restrict_address_families) + remove_ipc new_resource.remove_ipc if new_resource.property_is_set?(:remove_ipc) system_call_filter new_resource.system_call_filter if new_resource.property_is_set?(:system_call_filter) end diff --git a/cookbooks/tools/recipes/default.rb b/cookbooks/tools/recipes/default.rb index 05420a1d2..6ff55543f 100644 --- a/cookbooks/tools/recipes/default.rb +++ b/cookbooks/tools/recipes/default.rb @@ -38,6 +38,7 @@ package %w[ cron locales-all systemd-coredump + vim ] service "rsyslog" do diff --git a/cookbooks/web/recipes/frontend.rb b/cookbooks/web/recipes/frontend.rb index 8956b78bb..63dabd9c0 100644 --- a/cookbooks/web/recipes/frontend.rb +++ b/cookbooks/web/recipes/frontend.rb @@ -64,23 +64,37 @@ template "/etc/logrotate.d/apache2" do mode "644" end -service "rails-jobs@mailers" do - action [:enable, :start] - supports :restart => true - subscribes :restart, "rails_port[www.openstreetmap.org]" - subscribes :restart, "systemd_service[rails-jobs@]" -end +if %w[database_offline database_readonly].include?(node[:web][:status]) + service "rails-jobs@mailers" do + action :stop + end -service "rails-jobs@storage" do - action [:enable, :start] - supports :restart => true - subscribes :restart, "rails_port[www.openstreetmap.org]" - subscribes :restart, "systemd_service[rails-jobs@]" -end + service "rails-jobs@storage" do + action :stop + end + + service "rails-jobs@traces" do + action :stop + end +else + service "rails-jobs@mailers" do + action [:enable, :start] + supports :restart => true + subscribes :restart, "rails_port[www.openstreetmap.org]" + subscribes :restart, "systemd_service[rails-jobs@]" + end + + service "rails-jobs@storage" do + action [:enable, :start] + supports :restart => true + subscribes :restart, "rails_port[www.openstreetmap.org]" + subscribes :restart, "systemd_service[rails-jobs@]" + end -service "rails-jobs@traces" do - action [:enable, :start] - supports :restart => true - subscribes :restart, "rails_port[www.openstreetmap.org]" - subscribes :restart, "systemd_service[rails-jobs@]" + service "rails-jobs@traces" do + action [:enable, :start] + supports :restart => true + subscribes :restart, "rails_port[www.openstreetmap.org]" + subscribes :restart, "systemd_service[rails-jobs@]" + end end diff --git a/roles/db-master.rb b/roles/db-master.rb index ffea0f019..a107afa71 100644 --- a/roles/db-master.rb +++ b/roles/db-master.rb @@ -5,10 +5,8 @@ default_attributes( :postgresql => { :settings => { :defaults => { - :wal_level => "logical", :archive_mode => "on", :archive_command => "/usr/local/bin/openstreetmap-wal-g wal-push %p --walg-prevent-wal-overwrite=true", - :max_wal_senders => "10", :max_replication_slots => "1", :late_authentication_rules => [ { :database => "replication", :user => "replication", :address => "10.0.0.4/32" }, # snap-02 diff --git a/roles/db.rb b/roles/db.rb index 7ff450b2f..81aed8678 100644 --- a/roles/db.rb +++ b/roles/db.rb @@ -41,9 +41,11 @@ default_attributes( :listen_addresses => "*", :max_connections => "1500", :max_stack_depth => "7MB", + :wal_level => "logical", :checkpoint_segments => "32", :max_wal_size => "1536MB", :checkpoint_completion_target => "0.8", + :max_wal_senders => "10", :cpu_tuple_cost => "0.1", :log_min_duration_statement => "1000", :late_authentication_rules => [ diff --git a/roles/dev.rb b/roles/dev.rb index e048537fa..c41dda8dc 100644 --- a/roles/dev.rb +++ b/roles/dev.rb @@ -127,7 +127,7 @@ default_attributes( } }, :postgresql => { - :versions => ["14"], + :versions => ["15"], :settings => { :defaults => { :max_connections => "500", @@ -137,7 +137,7 @@ default_attributes( :max_stack_depth => "4MB", :effective_cache_size => "4GB" }, - "14" => { + "15" => { :port => "5432", :wal_level => "logical", :max_replication_slots => "1" diff --git a/roles/eddie.rb b/roles/eddie.rb index 40a105db5..0dbe9dd0a 100644 --- a/roles/eddie.rb +++ b/roles/eddie.rb @@ -37,5 +37,5 @@ default_attributes( run_list( "role[ucl]", - "role[db-master]" + "role[db-slave]" ) diff --git a/test/integration/db-base/serverspec/postgresql_spec.rb b/test/integration/db-base/serverspec/postgresql_spec.rb index b6e920c8f..84008f457 100644 --- a/test/integration/db-base/serverspec/postgresql_spec.rb +++ b/test/integration/db-base/serverspec/postgresql_spec.rb @@ -3,11 +3,11 @@ require "serverspec" # Required by serverspec set :backend, :exec -describe package("postgresql-9.5") do +describe package("postgresql-15") do it { should be_installed } end -describe service("postgresql@9.5-main") do +describe service("postgresql@15-main") do it { should be_enabled } it { should be_running } end diff --git a/test/integration/db-master/serverspec/postgresql_spec.rb b/test/integration/db-master/serverspec/postgresql_spec.rb index b6e920c8f..84008f457 100644 --- a/test/integration/db-master/serverspec/postgresql_spec.rb +++ b/test/integration/db-master/serverspec/postgresql_spec.rb @@ -3,11 +3,11 @@ require "serverspec" # Required by serverspec set :backend, :exec -describe package("postgresql-9.5") do +describe package("postgresql-15") do it { should be_installed } end -describe service("postgresql@9.5-main") do +describe service("postgresql@15-main") do it { should be_enabled } it { should be_running } end diff --git a/test/integration/db-slave/serverspec/postgresql_spec.rb b/test/integration/db-slave/serverspec/postgresql_spec.rb index b6e920c8f..84008f457 100644 --- a/test/integration/db-slave/serverspec/postgresql_spec.rb +++ b/test/integration/db-slave/serverspec/postgresql_spec.rb @@ -3,11 +3,11 @@ require "serverspec" # Required by serverspec set :backend, :exec -describe package("postgresql-9.5") do +describe package("postgresql-15") do it { should be_installed } end -describe service("postgresql@9.5-main") do +describe service("postgresql@15-main") do it { should be_enabled } it { should be_running } end