X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/25b07c5ab577080d2ae336435ad1a80d4c83851c..bb01eaa9176f18aacc218e8f706f987afd13595e:/cookbooks/prometheus/recipes/server.rb diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb index 155608926..45550368c 100644 --- a/cookbooks/prometheus/recipes/server.rb +++ b/cookbooks/prometheus/recipes/server.rb @@ -18,9 +18,8 @@ # include_recipe "apache" -include_recipe "apt" +include_recipe "apt::grafana" include_recipe "networking" -include_recipe "timescaledb" passwords = data_bag_item("prometheus", "passwords") tokens = data_bag_item("prometheus", "tokens") @@ -40,14 +39,16 @@ end prometheus_exporter "statuscake" do port 9595 + scrape_interval "5m" + scrape_timeout "2m" environment "STATUSCAKE_APIKEY" => tokens["statuscake"] end cache_dir = Chef::Config[:file_cache_path] -prometheus_version = "2.31.1" -alertmanager_version = "0.23.0" -karma_version = "0.105" +prometheus_version = "2.45.0" +alertmanager_version = "0.25.0" +karma_version = "0.114" directory "/opt/prometheus-server" do owner "root" @@ -55,128 +56,67 @@ directory "/opt/prometheus-server" do mode "755" end -remote_file "#{cache_dir}/prometheus.linux-amd64.tar.gz" do - source "https://github.com/prometheus/prometheus/releases/download/v#{prometheus_version}/prometheus-#{prometheus_version}.linux-amd64.tar.gz" +prometheus_arch = if arm? + "arm64" + else + "amd64" + end + +remote_file "#{cache_dir}/prometheus.linux.tar.gz" do + source "https://github.com/prometheus/prometheus/releases/download/v#{prometheus_version}/prometheus-#{prometheus_version}.linux-#{prometheus_arch}.tar.gz" owner "root" group "root" mode "644" backup false end -archive_file "#{cache_dir}/prometheus.linux-amd64.tar.gz" do +archive_file "#{cache_dir}/prometheus.linux.tar.gz" do action :nothing destination "/opt/prometheus-server/prometheus" overwrite true strip_components 1 owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]" + subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux.tar.gz]", :immediately end -remote_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do - source "https://github.com/prometheus/alertmanager/releases/download/v#{alertmanager_version}/alertmanager-#{alertmanager_version}.linux-amd64.tar.gz" +remote_file "#{cache_dir}/alertmanager.linux.tar.gz" do + source "https://github.com/prometheus/alertmanager/releases/download/v#{alertmanager_version}/alertmanager-#{alertmanager_version}.linux-#{prometheus_arch}.tar.gz" owner "root" group "root" mode "644" backup false end -archive_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do +archive_file "#{cache_dir}/alertmanager.linux.tar.gz" do action :nothing destination "/opt/prometheus-server/alertmanager" overwrite true strip_components 1 owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]" + subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux.tar.gz]", :immediately end -remote_file "#{cache_dir}/karma-linux-amd64.tar.gz" do - source "https://github.com/prymitive/karma/releases/download/v#{karma_version}/karma-linux-amd64.tar.gz" +remote_file "#{cache_dir}/karma-linux.tar.gz" do + source "https://github.com/prymitive/karma/releases/download/v#{karma_version}/karma-linux-#{prometheus_arch}.tar.gz" owner "root" group "root" mode "644" backup false end -archive_file "#{cache_dir}/karma-linux-amd64.tar.gz" do +archive_file "#{cache_dir}/karma-linux.tar.gz" do action :nothing destination "/opt/prometheus-server/karma" overwrite true owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/karma-linux-amd64.tar.gz]" -end - -promscale_version = "0.13.0" - -database_version = node[:timescaledb][:database_version] -database_cluster = "#{database_version}/main" - -package %W[ - prometheus - prometheus-alertmanager - promscale-extension-postgresql-#{database_version} -] - -postgresql_user "prometheus" do - cluster database_cluster - superuser true -end - -postgresql_database "promscale" do - cluster database_cluster - owner "prometheus" -end - -directory "/opt/promscale" do - owner "root" - group "root" - mode "755" -end - -directory "/opt/promscale/bin" do - owner "root" - group "root" - mode "755" -end - -remote_file "/opt/promscale/bin/promscale" do - action :create - source "https://github.com/timescale/promscale/releases/download/#{promscale_version}/promscale_#{promscale_version}_Linux_x86_64" - owner "root" - group "root" - mode "755" -end - -systemd_service "promscale" do - description "Promscale Connector" - type "simple" - user "prometheus" - exec_start "/opt/promscale/bin/promscale --db.uri postgresql:///promscale?host=/run/postgresql&port=5432 --db.connections-max 400" - limit_nofile 16384 - private_tmp true - protect_system "strict" - protect_home true - no_new_privileges true -end - -if node[:prometheus][:promscale] - service "promscale" do - action [:enable, :start] - subscribes :restart, "remote_file[/opt/promscale/bin/promscale]" - subscribes :restart, "systemd_service[promscale]" - end -else - service "promscale" do - action [:disable, :stop] - end + subscribes :extract, "remote_file[#{cache_dir}/karma-linux.tar.gz]", :immediately end search(:node, "roles:gateway") do |gateway| - allowed_ips = gateway.interfaces(:role => :internal).map do |interface| - "#{interface[:network]}/#{interface[:prefix]}" - end + allowed_ips = gateway.ipaddresses(:role => :internal).map(&:subnet) node.default[:networking][:wireguard][:peers] << { :public_key => gateway[:networking][:wireguard][:public_key], @@ -186,6 +126,7 @@ search(:node, "roles:gateway") do |gateway| end jobs = {} +junos_targets = [] snmp_targets = [] search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client| @@ -202,13 +143,17 @@ search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client name = exporter[:name] address = exporter[:address] sni = exporter[:sni] + labels = Array(exporter[:labels]) scrape_interval = exporter[:scrape_interval] + scrape_timeout = exporter[:scrape_timeout] metric_relabel = exporter[:metric_relabel] || [] else name = key address = exporter sni = nil + labels = [] scrape_interval = nil + scrape_timeout = nil metric_relabel = [] end @@ -217,11 +162,22 @@ search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client :address => address, :sni => sni, :instance => client.name.split(".").first, + :labels => labels, :scrape_interval => scrape_interval, + :scrape_timeout => scrape_timeout, :metric_relabel => metric_relabel } end + Hash(client[:prometheus][:junos]).each do |instance, details| + junos_targets << { + :instance => instance, + :target => details[:address], + :address => client[:prometheus][:addresses]["junos"], + :labels => Array(details[:labels]) + } + end + Hash(client[:prometheus][:snmp]).each do |instance, details| snmp_targets << { :instance => instance, @@ -259,10 +215,12 @@ prometheus_exporter "ssl" do register_target false end +package "prometheus" + systemd_service "prometheus-executable" do service "prometheus" dropin "executable" - exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=30d" + exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=540d" timeout_stop_sec 300 notifies :restart, "service[prometheus]" end @@ -272,7 +230,7 @@ template "/etc/prometheus/prometheus.yml" do owner "root" group "root" mode "644" - variables :jobs => jobs, :snmp_targets => snmp_targets, :certificates => certificates + variables :jobs => jobs, :junos_targets => junos_targets, :snmp_targets => snmp_targets, :certificates => certificates end template "/etc/prometheus/alert_rules.yml" do @@ -286,16 +244,24 @@ service "prometheus" do action [:enable, :start] subscribes :reload, "template[/etc/prometheus/prometheus.yml]" subscribes :reload, "template[/etc/prometheus/alert_rules.yml]" - subscribes :restart, "archive_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]" + subscribes :restart, "archive_file[#{cache_dir}/prometheus.linux.tar.gz]" end -systemd_service "prometheus-alertmanager-executable" do - service "prometheus-alertmanager" - dropin "executable" +systemd_service "prometheus-alertmanager" do + description "Prometheus alert manager" + type "simple" + user "prometheus" exec_start "/opt/prometheus-server/alertmanager/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/prometheus/alertmanager --web.external-url=https://prometheus.openstreetmap.org/alertmanager" + exec_reload "/bin/kill -HUP $MAINPID" + timeout_stop_sec 20 + restart "on-failure" notifies :restart, "service[prometheus-alertmanager]" end +link "/usr/local/bin/promtool" do + to "/opt/prometheus-server/prometheus/promtool" +end + template "/etc/prometheus/alertmanager.yml" do source "alertmanager.yml.erb" owner "root" @@ -303,19 +269,36 @@ template "/etc/prometheus/alertmanager.yml" do mode "644" end +directory "/var/lib/prometheus/alertmanager" do + owner "prometheus" + group "prometheus" + mode "755" +end + service "prometheus-alertmanager" do action [:enable, :start] subscribes :reload, "template[/etc/prometheus/alertmanager.yml]" - subscribes :restart, "archive_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]" + subscribes :restart, "systemd_service[prometheus-alertmanager]" + subscribes :restart, "archive_file[#{cache_dir}/alertmanager.linux.tar.gz]" end -template "/etc/prometheus/amtool.yml" do +directory "/etc/amtool" do + owner "root" + group "root" + mode "755" +end + +template "/etc/amtool/config.yml" do source "amtool.yml.erb" owner "root" group "root" mode "644" end +link "/usr/local/bin/amtool" do + to "/opt/prometheus-server/alertmanager/amtool" +end + template "/etc/prometheus/karma.yml" do source "karma.yml.erb" owner "root" @@ -326,19 +309,16 @@ end systemd_service "prometheus-karma" do description "Alert dashboard for Prometheus Alertmanager" user "prometheus" - exec_start "/opt/prometheus-server/karma/karma-linux-amd64 --config.file=/etc/prometheus/karma.yml" - private_tmp true - private_devices true - protect_system "full" - protect_home true - no_new_privileges true + exec_start "/opt/prometheus-server/karma/karma-linux-#{prometheus_arch} --config.file=/etc/prometheus/karma.yml" + sandbox :enable_network => true restart "on-failure" end service "prometheus-karma" do action [:enable, :start] - subscribes :reload, "template[/etc/prometheus/karma.yml]" - subscribes :restart, "archive_file[#{cache_dir}/karma-linux-amd64.tar.gz]" + subscribes :restart, "template[/etc/prometheus/karma.yml]" + subscribes :restart, "archive_file[#{cache_dir}/karma-linux.tar.gz]" + subscribes :restart, "systemd_service[prometheus-karma]" end package "grafana-enterprise" @@ -358,6 +338,7 @@ end apache_module "alias" apache_module "proxy_http" +apache_module "proxy_wstunnel" ssl_certificate "prometheus.openstreetmap.org" do domains ["prometheus.openstreetmap.org", "prometheus.osm.org"]