X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/09b410b27cd5eae901ae9898835038701513743f..1eb96a83e801132a0ee55049487d1bb42e1bcc6e:/cookbooks/prometheus/recipes/server.rb diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb index f0c4a8821..414819160 100644 --- a/cookbooks/prometheus/recipes/server.rb +++ b/cookbooks/prometheus/recipes/server.rb @@ -18,9 +18,8 @@ # include_recipe "apache" -include_recipe "apt" +include_recipe "apt::grafana" include_recipe "networking" -include_recipe "timescaledb" passwords = data_bag_item("prometheus", "passwords") tokens = data_bag_item("prometheus", "tokens") @@ -28,16 +27,28 @@ admins = data_bag_item("apache", "admins") prometheus_exporter "fastly" do port 8080 - listen_switch "endpoint" - listen_type "url" + listen_switch "listen" environment "FASTLY_API_TOKEN" => tokens["fastly"] end +prometheus_exporter "fastly_healthcheck" do + port 9696 + scrape_interval "1m" + environment "FASTLY_API_TOKEN" => tokens["fastly"] +end + +prometheus_exporter "statuscake" do + port 9595 + scrape_interval "5m" + scrape_timeout "2m" + environment "STATUSCAKE_APIKEY" => tokens["statuscake"] +end + cache_dir = Chef::Config[:file_cache_path] -prometheus_version = "2.31.1" -alertmanager_version = "0.23.0" -karma_version = "0.93" +prometheus_version = "2.44.0" +alertmanager_version = "0.25.0" +karma_version = "0.114" directory "/opt/prometheus-server" do owner "root" @@ -60,7 +71,7 @@ archive_file "#{cache_dir}/prometheus.linux-amd64.tar.gz" do strip_components 1 owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]" + subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]", :immediately end remote_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do @@ -78,7 +89,7 @@ archive_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do strip_components 1 owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]" + subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]", :immediately end remote_file "#{cache_dir}/karma-linux-amd64.tar.gz" do @@ -95,110 +106,11 @@ archive_file "#{cache_dir}/karma-linux-amd64.tar.gz" do overwrite true owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/karma-linux-amd64.tar.gz]" -end - -package %w[ - prometheus - prometheus-alertmanager -] - -promscale_version = "0.6.2" - -database_version = node[:timescaledb][:database_version] -database_cluster = "#{database_version}/main" - -postgresql_user "prometheus" do - cluster database_cluster - superuser true -end - -postgresql_database "promscale" do - cluster database_cluster - owner "prometheus" -end - -directory "/opt/promscale" do - owner "root" - group "root" - mode "755" -end - -cookbook_file "/usr/lib/postgresql/#{database_version}/lib/promscale.so" do - source "postgresql-#{database_version}-promscale.so" - owner "root" - group "root" - mode "644" -end - -directory "/opt/promscale/bin" do - owner "root" - group "root" - mode "755" -end - -remote_file "/opt/promscale/bin/promscale" do - action :create - source "https://github.com/timescale/promscale/releases/download/#{promscale_version}/promscale_#{promscale_version}_Linux_x86_64" - owner "root" - group "root" - mode "755" -end - -systemd_service "promscale" do - description "Promscale Connector" - type "simple" - user "prometheus" - exec_start "/opt/promscale/bin/promscale --db-uri postgresql:///promscale?host=/run/postgresql&port=5432 --db-connections-max 400" - # exec_start lazy { "/opt/promscale/bin/promscale --db-host /run/postgresql --db-port #{node[:postgresql][:clusters][database_cluster][:port]} --db-user prometheus --db-name promscale --db-max-connections 400" } - limit_nofile 16384 - private_tmp true - protect_system "strict" - protect_home true - no_new_privileges true -end - -systemd_service "promscale-maintenance" do - description "Promscale Maintenance" - type "simple" - user "prometheus" - exec_start "/usr/bin/psql --command='CALL prom_api.execute_maintenance()' promscale" - private_tmp true - protect_system "strict" - protect_home true - no_new_privileges true -end - -systemd_timer "promscale-maintenance" do - description "Promscale Maintenace" - on_active_sec 1800 - on_unit_inactive_sec 1800 -end - -if node[:prometheus][:promscale] - service "promscale" do - action [:enable, :start] - subscribes :restart, "remote_file[/opt/promscale/bin/promscale]" - subscribes :restart, "systemd_service[promscale]" - end - - service "promscale-maintenance.timer" do - action [:enable, :start] - end -else - service "promscale" do - action [:disable, :stop] - end - - service "promscale-maintenance.timer" do - action [:disable, :stop] - end + subscribes :extract, "remote_file[#{cache_dir}/karma-linux-amd64.tar.gz]", :immediately end search(:node, "roles:gateway") do |gateway| - allowed_ips = gateway.interfaces(:role => :internal).map do |interface| - "#{interface[:network]}/#{interface[:prefix]}" - end + allowed_ips = gateway.ipaddresses(:role => :internal).map(&:subnet) node.default[:networking][:wireguard][:peers] << { :public_key => gateway[:networking][:wireguard][:public_key], @@ -223,17 +135,26 @@ search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client if exporter.is_a?(Hash) name = exporter[:name] address = exporter[:address] + sni = exporter[:sni] + scrape_interval = exporter[:scrape_interval] + scrape_timeout = exporter[:scrape_timeout] metric_relabel = exporter[:metric_relabel] || [] else name = key address = exporter + sni = nil + scrape_interval = nil + scrape_timeout = nil metric_relabel = [] end jobs[name] ||= [] jobs[name] << { :address => address, + :sni => sni, :instance => client.name.split(".").first, + :scrape_interval => scrape_interval, + :scrape_timeout => scrape_timeout, :metric_relabel => metric_relabel } end @@ -242,7 +163,7 @@ search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client snmp_targets << { :instance => instance, :target => details[:address], - :module => details[:module], + :modules => details[:modules], :address => client[:prometheus][:addresses]["snmp"], :labels => Array(details[:labels]) } @@ -278,7 +199,8 @@ end systemd_service "prometheus-executable" do service "prometheus" dropin "executable" - exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2" + exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=540d" + timeout_stop_sec 300 notifies :restart, "service[prometheus]" end @@ -304,13 +226,21 @@ service "prometheus" do subscribes :restart, "archive_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]" end -systemd_service "prometheus-alertmanager-executable" do - service "prometheus-alertmanager" - dropin "executable" +systemd_service "prometheus-alertmanager" do + description "Prometheus alert manager" + type "simple" + user "prometheus" exec_start "/opt/prometheus-server/alertmanager/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/prometheus/alertmanager --web.external-url=https://prometheus.openstreetmap.org/alertmanager" + exec_reload "/bin/kill -HUP $MAINPID" + timeout_stop_sec 20 + restart "on-failure" notifies :restart, "service[prometheus-alertmanager]" end +link "/usr/local/bin/promtool" do + to "/opt/prometheus-server/prometheus/promtool" +end + template "/etc/prometheus/alertmanager.yml" do source "alertmanager.yml.erb" owner "root" @@ -318,19 +248,36 @@ template "/etc/prometheus/alertmanager.yml" do mode "644" end +directory "/var/lib/prometheus/alertmanager" do + owner "prometheus" + group "prometheus" + mode "755" +end + service "prometheus-alertmanager" do action [:enable, :start] subscribes :reload, "template[/etc/prometheus/alertmanager.yml]" + subscribes :restart, "systemd_service[prometheus-alertmanager]" subscribes :restart, "archive_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]" end -template "/etc/prometheus/amtool.yml" do +directory "/etc/amtool" do + owner "root" + group "root" + mode "755" +end + +template "/etc/amtool/config.yml" do source "amtool.yml.erb" owner "root" group "root" mode "644" end +link "/usr/local/bin/amtool" do + to "/opt/prometheus-server/alertmanager/amtool" +end + template "/etc/prometheus/karma.yml" do source "karma.yml.erb" owner "root" @@ -342,18 +289,15 @@ systemd_service "prometheus-karma" do description "Alert dashboard for Prometheus Alertmanager" user "prometheus" exec_start "/opt/prometheus-server/karma/karma-linux-amd64 --config.file=/etc/prometheus/karma.yml" - private_tmp true - private_devices true - protect_system "full" - protect_home true - no_new_privileges true + sandbox :enable_network => true restart "on-failure" end service "prometheus-karma" do action [:enable, :start] - subscribes :reload, "template[/etc/prometheus/karma.yml]" + subscribes :restart, "template[/etc/prometheus/karma.yml]" subscribes :restart, "archive_file[#{cache_dir}/karma-linux-amd64.tar.gz]" + subscribes :restart, "systemd_service[prometheus-karma]" end package "grafana-enterprise" @@ -373,6 +317,7 @@ end apache_module "alias" apache_module "proxy_http" +apache_module "proxy_wstunnel" ssl_certificate "prometheus.openstreetmap.org" do domains ["prometheus.openstreetmap.org", "prometheus.osm.org"]