X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/acd1a016748bd3f8b47b370b9a7c15cbf44162f1..5fde77fb90280da2c008bb5e8e83f6635b999715:/cookbooks/prometheus/recipes/server.rb?ds=inline diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb index e10e9e4ce..45550368c 100644 --- a/cookbooks/prometheus/recipes/server.rb +++ b/cookbooks/prometheus/recipes/server.rb @@ -18,14 +18,116 @@ # include_recipe "apache" -include_recipe "apt" +include_recipe "apt::grafana" include_recipe "networking" passwords = data_bag_item("prometheus", "passwords") +tokens = data_bag_item("prometheus", "tokens") +admins = data_bag_item("apache", "admins") -package "prometheus" +prometheus_exporter "fastly" do + port 8080 + listen_switch "listen" + environment "FASTLY_API_TOKEN" => tokens["fastly"] +end + +prometheus_exporter "fastly_healthcheck" do + port 9696 + scrape_interval "1m" + environment "FASTLY_API_TOKEN" => tokens["fastly"] +end + +prometheus_exporter "statuscake" do + port 9595 + scrape_interval "5m" + scrape_timeout "2m" + environment "STATUSCAKE_APIKEY" => tokens["statuscake"] +end + +cache_dir = Chef::Config[:file_cache_path] + +prometheus_version = "2.45.0" +alertmanager_version = "0.25.0" +karma_version = "0.114" + +directory "/opt/prometheus-server" do + owner "root" + group "root" + mode "755" +end + +prometheus_arch = if arm? + "arm64" + else + "amd64" + end + +remote_file "#{cache_dir}/prometheus.linux.tar.gz" do + source "https://github.com/prometheus/prometheus/releases/download/v#{prometheus_version}/prometheus-#{prometheus_version}.linux-#{prometheus_arch}.tar.gz" + owner "root" + group "root" + mode "644" + backup false +end + +archive_file "#{cache_dir}/prometheus.linux.tar.gz" do + action :nothing + destination "/opt/prometheus-server/prometheus" + overwrite true + strip_components 1 + owner "root" + group "root" + subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux.tar.gz]", :immediately +end + +remote_file "#{cache_dir}/alertmanager.linux.tar.gz" do + source "https://github.com/prometheus/alertmanager/releases/download/v#{alertmanager_version}/alertmanager-#{alertmanager_version}.linux-#{prometheus_arch}.tar.gz" + owner "root" + group "root" + mode "644" + backup false +end + +archive_file "#{cache_dir}/alertmanager.linux.tar.gz" do + action :nothing + destination "/opt/prometheus-server/alertmanager" + overwrite true + strip_components 1 + owner "root" + group "root" + subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux.tar.gz]", :immediately +end + +remote_file "#{cache_dir}/karma-linux.tar.gz" do + source "https://github.com/prymitive/karma/releases/download/v#{karma_version}/karma-linux-#{prometheus_arch}.tar.gz" + owner "root" + group "root" + mode "644" + backup false +end + +archive_file "#{cache_dir}/karma-linux.tar.gz" do + action :nothing + destination "/opt/prometheus-server/karma" + overwrite true + owner "root" + group "root" + subscribes :extract, "remote_file[#{cache_dir}/karma-linux.tar.gz]", :immediately +end + +search(:node, "roles:gateway") do |gateway| + allowed_ips = gateway.ipaddresses(:role => :internal).map(&:subnet) + + node.default[:networking][:wireguard][:peers] << { + :public_key => gateway[:networking][:wireguard][:public_key], + :allowed_ips => allowed_ips, + :endpoint => "#{gateway.name}:51820" + } +end jobs = {} +junos_targets = [] +snmp_targets = [] search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client| if client[:prometheus][:mode] == "wireguard" @@ -36,23 +138,187 @@ search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client } end - client[:prometheus][:exporters].each do |name, address| + client[:prometheus][:exporters].each do |key, exporter| + if exporter.is_a?(Hash) + name = exporter[:name] + address = exporter[:address] + sni = exporter[:sni] + labels = Array(exporter[:labels]) + scrape_interval = exporter[:scrape_interval] + scrape_timeout = exporter[:scrape_timeout] + metric_relabel = exporter[:metric_relabel] || [] + else + name = key + address = exporter + sni = nil + labels = [] + scrape_interval = nil + scrape_timeout = nil + metric_relabel = [] + end + jobs[name] ||= [] - jobs[name] << { :address => address, :name => client.name } + jobs[name] << { + :address => address, + :sni => sni, + :instance => client.name.split(".").first, + :labels => labels, + :scrape_interval => scrape_interval, + :scrape_timeout => scrape_timeout, + :metric_relabel => metric_relabel + } + end + + Hash(client[:prometheus][:junos]).each do |instance, details| + junos_targets << { + :instance => instance, + :target => details[:address], + :address => client[:prometheus][:addresses]["junos"], + :labels => Array(details[:labels]) + } + end + + Hash(client[:prometheus][:snmp]).each do |instance, details| + snmp_targets << { + :instance => instance, + :target => details[:address], + :modules => details[:modules], + :address => client[:prometheus][:addresses]["snmp"], + :labels => Array(details[:labels]) + } + end +end + +certificates = search(:node, "letsencrypt:certificates").each_with_object({}) do |n, c| + n[:letsencrypt][:certificates].each do |name, details| + c[name] ||= details.merge(:nodes => []) + + c[name][:nodes] << { + :name => n[:fqdn], + :address => n.external_ipaddress || n.internal_ipaddress + } end end +template "/etc/prometheus/ssl.yml" do + source "ssl.yml.erb" + owner "root" + group "root" + mode "644" + variables :certificates => certificates +end + +prometheus_exporter "ssl" do + address "127.0.0.1" + port 9219 + options "--config.file=/etc/prometheus/ssl.yml" + register_target false +end + +package "prometheus" + +systemd_service "prometheus-executable" do + service "prometheus" + dropin "executable" + exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=540d" + timeout_stop_sec 300 + notifies :restart, "service[prometheus]" +end + template "/etc/prometheus/prometheus.yml" do source "prometheus.yml.erb" owner "root" group "root" mode "644" - variables :jobs => jobs + variables :jobs => jobs, :junos_targets => junos_targets, :snmp_targets => snmp_targets, :certificates => certificates +end + +template "/etc/prometheus/alert_rules.yml" do + source "alert_rules.yml.erb" + owner "root" + group "root" + mode "644" end service "prometheus" do action [:enable, :start] subscribes :reload, "template[/etc/prometheus/prometheus.yml]" + subscribes :reload, "template[/etc/prometheus/alert_rules.yml]" + subscribes :restart, "archive_file[#{cache_dir}/prometheus.linux.tar.gz]" +end + +systemd_service "prometheus-alertmanager" do + description "Prometheus alert manager" + type "simple" + user "prometheus" + exec_start "/opt/prometheus-server/alertmanager/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/prometheus/alertmanager --web.external-url=https://prometheus.openstreetmap.org/alertmanager" + exec_reload "/bin/kill -HUP $MAINPID" + timeout_stop_sec 20 + restart "on-failure" + notifies :restart, "service[prometheus-alertmanager]" +end + +link "/usr/local/bin/promtool" do + to "/opt/prometheus-server/prometheus/promtool" +end + +template "/etc/prometheus/alertmanager.yml" do + source "alertmanager.yml.erb" + owner "root" + group "root" + mode "644" +end + +directory "/var/lib/prometheus/alertmanager" do + owner "prometheus" + group "prometheus" + mode "755" +end + +service "prometheus-alertmanager" do + action [:enable, :start] + subscribes :reload, "template[/etc/prometheus/alertmanager.yml]" + subscribes :restart, "systemd_service[prometheus-alertmanager]" + subscribes :restart, "archive_file[#{cache_dir}/alertmanager.linux.tar.gz]" +end + +directory "/etc/amtool" do + owner "root" + group "root" + mode "755" +end + +template "/etc/amtool/config.yml" do + source "amtool.yml.erb" + owner "root" + group "root" + mode "644" +end + +link "/usr/local/bin/amtool" do + to "/opt/prometheus-server/alertmanager/amtool" +end + +template "/etc/prometheus/karma.yml" do + source "karma.yml.erb" + owner "root" + group "root" + mode "644" +end + +systemd_service "prometheus-karma" do + description "Alert dashboard for Prometheus Alertmanager" + user "prometheus" + exec_start "/opt/prometheus-server/karma/karma-linux-#{prometheus_arch} --config.file=/etc/prometheus/karma.yml" + sandbox :enable_network => true + restart "on-failure" +end + +service "prometheus-karma" do + action [:enable, :start] + subscribes :restart, "template[/etc/prometheus/karma.yml]" + subscribes :restart, "archive_file[#{cache_dir}/karma-linux.tar.gz]" + subscribes :restart, "systemd_service[prometheus-karma]" end package "grafana-enterprise" @@ -72,6 +338,7 @@ end apache_module "alias" apache_module "proxy_http" +apache_module "proxy_wstunnel" ssl_certificate "prometheus.openstreetmap.org" do domains ["prometheus.openstreetmap.org", "prometheus.osm.org"] @@ -80,4 +347,12 @@ end apache_site "prometheus.openstreetmap.org" do template "apache.erb" + variables :admin_hosts => admins["hosts"] +end + +template "/etc/cron.daily/prometheus-backup" do + source "backup.cron.erb" + owner "root" + group "root" + mode "750" end