X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/93b82f6239bf810ecdb96241c7680628f5a7ebc9..5fde77fb90280da2c008bb5e8e83f6635b999715:/cookbooks/prometheus/recipes/server.rb diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb index 675a206ec..45550368c 100644 --- a/cookbooks/prometheus/recipes/server.rb +++ b/cookbooks/prometheus/recipes/server.rb @@ -18,9 +18,8 @@ # include_recipe "apache" -include_recipe "apt" +include_recipe "apt::grafana" include_recipe "networking" -include_recipe "timescaledb" passwords = data_bag_item("prometheus", "passwords") tokens = data_bag_item("prometheus", "tokens") @@ -28,15 +27,28 @@ admins = data_bag_item("apache", "admins") prometheus_exporter "fastly" do port 8080 - listen_switch "endpoint" - listen_type "url" + listen_switch "listen" environment "FASTLY_API_TOKEN" => tokens["fastly"] end +prometheus_exporter "fastly_healthcheck" do + port 9696 + scrape_interval "1m" + environment "FASTLY_API_TOKEN" => tokens["fastly"] +end + +prometheus_exporter "statuscake" do + port 9595 + scrape_interval "5m" + scrape_timeout "2m" + environment "STATUSCAKE_APIKEY" => tokens["statuscake"] +end + cache_dir = Chef::Config[:file_cache_path] -prometheus_version = "2.31.1" -alertmanager_version = "0.23.0" +prometheus_version = "2.45.0" +alertmanager_version = "0.25.0" +karma_version = "0.114" directory "/opt/prometheus-server" do owner "root" @@ -44,143 +56,67 @@ directory "/opt/prometheus-server" do mode "755" end -remote_file "#{cache_dir}/prometheus.linux-amd64.tar.gz" do - source "https://github.com/prometheus/prometheus/releases/download/v#{prometheus_version}/prometheus-#{prometheus_version}.linux-amd64.tar.gz" +prometheus_arch = if arm? + "arm64" + else + "amd64" + end + +remote_file "#{cache_dir}/prometheus.linux.tar.gz" do + source "https://github.com/prometheus/prometheus/releases/download/v#{prometheus_version}/prometheus-#{prometheus_version}.linux-#{prometheus_arch}.tar.gz" owner "root" group "root" mode "644" backup false end -archive_file "#{cache_dir}/prometheus.linux-amd64.tar.gz" do +archive_file "#{cache_dir}/prometheus.linux.tar.gz" do action :nothing destination "/opt/prometheus-server/prometheus" overwrite true strip_components 1 owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]" + subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux.tar.gz]", :immediately end -remote_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do - source "https://github.com/prometheus/alertmanager/releases/download/v#{alertmanager_version}/alertmanager-#{alertmanager_version}.linux-amd64.tar.gz" +remote_file "#{cache_dir}/alertmanager.linux.tar.gz" do + source "https://github.com/prometheus/alertmanager/releases/download/v#{alertmanager_version}/alertmanager-#{alertmanager_version}.linux-#{prometheus_arch}.tar.gz" owner "root" group "root" mode "644" backup false end -archive_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do +archive_file "#{cache_dir}/alertmanager.linux.tar.gz" do action :nothing destination "/opt/prometheus-server/alertmanager" overwrite true strip_components 1 owner "root" group "root" - subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]" -end - -package %w[ - prometheus - prometheus-alertmanager -] - -promscale_version = "0.6.2" - -database_version = node[:timescaledb][:database_version] -database_cluster = "#{database_version}/main" - -postgresql_user "prometheus" do - cluster database_cluster - superuser true + subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux.tar.gz]", :immediately end -postgresql_database "promscale" do - cluster database_cluster - owner "prometheus" -end - -directory "/opt/promscale" do - owner "root" - group "root" - mode "755" -end - -cookbook_file "/usr/lib/postgresql/#{database_version}/lib/promscale.so" do - source "postgresql-#{database_version}-promscale.so" +remote_file "#{cache_dir}/karma-linux.tar.gz" do + source "https://github.com/prymitive/karma/releases/download/v#{karma_version}/karma-linux-#{prometheus_arch}.tar.gz" owner "root" group "root" mode "644" + backup false end -directory "/opt/promscale/bin" do - owner "root" - group "root" - mode "755" -end - -remote_file "/opt/promscale/bin/promscale" do - action :create - source "https://github.com/timescale/promscale/releases/download/#{promscale_version}/promscale_#{promscale_version}_Linux_x86_64" +archive_file "#{cache_dir}/karma-linux.tar.gz" do + action :nothing + destination "/opt/prometheus-server/karma" + overwrite true owner "root" group "root" - mode "755" -end - -systemd_service "promscale" do - description "Promscale Connector" - type "simple" - user "prometheus" - exec_start "/opt/promscale/bin/promscale --db-uri postgresql:///promscale?host=/run/postgresql&port=5432 --db-connections-max 400" - # exec_start lazy { "/opt/promscale/bin/promscale --db-host /run/postgresql --db-port #{node[:postgresql][:clusters][database_cluster][:port]} --db-user prometheus --db-name promscale --db-max-connections 400" } - limit_nofile 16384 - private_tmp true - protect_system "strict" - protect_home true - no_new_privileges true -end - -systemd_service "promscale-maintenance" do - description "Promscale Maintenance" - type "simple" - user "prometheus" - exec_start "/usr/bin/psql --command='CALL prom_api.execute_maintenance()' promscale" - private_tmp true - protect_system "strict" - protect_home true - no_new_privileges true -end - -systemd_timer "promscale-maintenance" do - description "Promscale Maintenace" - on_active_sec 1800 - on_unit_inactive_sec 1800 -end - -if node[:prometheus][:promscale] - service "promscale" do - action [:enable, :start] - subscribes :restart, "remote_file[/opt/promscale/bin/promscale]" - subscribes :restart, "systemd_service[promscale]" - end - - service "promscale-maintenance.timer" do - action [:enable, :start] - end -else - service "promscale" do - action [:disable, :stop] - end - - service "promscale-maintenance.timer" do - action [:disable, :stop] - end + subscribes :extract, "remote_file[#{cache_dir}/karma-linux.tar.gz]", :immediately end search(:node, "roles:gateway") do |gateway| - allowed_ips = gateway.interfaces(:role => :internal).map do |interface| - "#{interface[:network]}/#{interface[:prefix]}" - end + allowed_ips = gateway.ipaddresses(:role => :internal).map(&:subnet) node.default[:networking][:wireguard][:peers] << { :public_key => gateway[:networking][:wireguard][:public_key], @@ -190,6 +126,7 @@ search(:node, "roles:gateway") do |gateway| end jobs = {} +junos_targets = [] snmp_targets = [] search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client| @@ -205,26 +142,47 @@ search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client if exporter.is_a?(Hash) name = exporter[:name] address = exporter[:address] + sni = exporter[:sni] + labels = Array(exporter[:labels]) + scrape_interval = exporter[:scrape_interval] + scrape_timeout = exporter[:scrape_timeout] metric_relabel = exporter[:metric_relabel] || [] else name = key address = exporter + sni = nil + labels = [] + scrape_interval = nil + scrape_timeout = nil metric_relabel = [] end jobs[name] ||= [] jobs[name] << { :address => address, + :sni => sni, :instance => client.name.split(".").first, + :labels => labels, + :scrape_interval => scrape_interval, + :scrape_timeout => scrape_timeout, :metric_relabel => metric_relabel } end + Hash(client[:prometheus][:junos]).each do |instance, details| + junos_targets << { + :instance => instance, + :target => details[:address], + :address => client[:prometheus][:addresses]["junos"], + :labels => Array(details[:labels]) + } + end + Hash(client[:prometheus][:snmp]).each do |instance, details| snmp_targets << { :instance => instance, :target => details[:address], - :module => details[:module], + :modules => details[:modules], :address => client[:prometheus][:addresses]["snmp"], :labels => Array(details[:labels]) } @@ -257,10 +215,13 @@ prometheus_exporter "ssl" do register_target false end +package "prometheus" + systemd_service "prometheus-executable" do service "prometheus" dropin "executable" - exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2" + exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=540d" + timeout_stop_sec 300 notifies :restart, "service[prometheus]" end @@ -269,7 +230,7 @@ template "/etc/prometheus/prometheus.yml" do owner "root" group "root" mode "644" - variables :jobs => jobs, :snmp_targets => snmp_targets, :certificates => certificates + variables :jobs => jobs, :junos_targets => junos_targets, :snmp_targets => snmp_targets, :certificates => certificates end template "/etc/prometheus/alert_rules.yml" do @@ -283,15 +244,24 @@ service "prometheus" do action [:enable, :start] subscribes :reload, "template[/etc/prometheus/prometheus.yml]" subscribes :reload, "template[/etc/prometheus/alert_rules.yml]" + subscribes :restart, "archive_file[#{cache_dir}/prometheus.linux.tar.gz]" end -systemd_service "prometheus-alertmanager-executable" do - service "prometheus-alertmanager" - dropin "executable" +systemd_service "prometheus-alertmanager" do + description "Prometheus alert manager" + type "simple" + user "prometheus" exec_start "/opt/prometheus-server/alertmanager/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/prometheus/alertmanager --web.external-url=https://prometheus.openstreetmap.org/alertmanager" + exec_reload "/bin/kill -HUP $MAINPID" + timeout_stop_sec 20 + restart "on-failure" notifies :restart, "service[prometheus-alertmanager]" end +link "/usr/local/bin/promtool" do + to "/opt/prometheus-server/prometheus/promtool" +end + template "/etc/prometheus/alertmanager.yml" do source "alertmanager.yml.erb" owner "root" @@ -299,18 +269,58 @@ template "/etc/prometheus/alertmanager.yml" do mode "644" end +directory "/var/lib/prometheus/alertmanager" do + owner "prometheus" + group "prometheus" + mode "755" +end + service "prometheus-alertmanager" do action [:enable, :start] subscribes :reload, "template[/etc/prometheus/alertmanager.yml]" + subscribes :restart, "systemd_service[prometheus-alertmanager]" + subscribes :restart, "archive_file[#{cache_dir}/alertmanager.linux.tar.gz]" +end + +directory "/etc/amtool" do + owner "root" + group "root" + mode "755" end -template "/etc/prometheus/amtool.yml" do +template "/etc/amtool/config.yml" do source "amtool.yml.erb" owner "root" group "root" mode "644" end +link "/usr/local/bin/amtool" do + to "/opt/prometheus-server/alertmanager/amtool" +end + +template "/etc/prometheus/karma.yml" do + source "karma.yml.erb" + owner "root" + group "root" + mode "644" +end + +systemd_service "prometheus-karma" do + description "Alert dashboard for Prometheus Alertmanager" + user "prometheus" + exec_start "/opt/prometheus-server/karma/karma-linux-#{prometheus_arch} --config.file=/etc/prometheus/karma.yml" + sandbox :enable_network => true + restart "on-failure" +end + +service "prometheus-karma" do + action [:enable, :start] + subscribes :restart, "template[/etc/prometheus/karma.yml]" + subscribes :restart, "archive_file[#{cache_dir}/karma-linux.tar.gz]" + subscribes :restart, "systemd_service[prometheus-karma]" +end + package "grafana-enterprise" template "/etc/grafana/grafana.ini" do @@ -328,6 +338,7 @@ end apache_module "alias" apache_module "proxy_http" +apache_module "proxy_wstunnel" ssl_certificate "prometheus.openstreetmap.org" do domains ["prometheus.openstreetmap.org", "prometheus.osm.org"]