From: Tom Hughes <tom@compton.nu>
Date: Sun, 7 Mar 2021 12:14:52 +0000 (+0000)
Subject: Add prometheus SNMP monitoring of switch and PDUs in Amsterdam
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/bb670c9eb8501639d3073dfa54c2d921ca8da5ee

Add prometheus SNMP monitoring of switch and PDUs in Amsterdam
---

diff --git a/cookbooks/prometheus/attributes/default.rb b/cookbooks/prometheus/attributes/default.rb
index 56781b4dc..0200e1ca1 100644
--- a/cookbooks/prometheus/attributes/default.rb
+++ b/cookbooks/prometheus/attributes/default.rb
@@ -1,4 +1,6 @@
+default[:prometheus][:addresses] = {}
 default[:prometheus][:exporters] = {}
+default[:prometheus][:snmp] = {}
 
 if node[:recipes].include?("prometheus::server")
   default[:apt][:sources] |= ["grafana"]
diff --git a/cookbooks/prometheus/recipes/default.rb b/cookbooks/prometheus/recipes/default.rb
index 2ad1d69aa..6e88362a4 100644
--- a/cookbooks/prometheus/recipes/default.rb
+++ b/cookbooks/prometheus/recipes/default.rb
@@ -113,3 +113,11 @@ prometheus_exporter "node" do
   ]
   metric_relabel metric_relabel
 end
+
+unless node[:prometheus][:snmp].empty?
+  prometheus_exporter "snmp" do
+    port 9116
+    options "--config.file=/opt/prometheus/exporters/snmp/snmp.yml"
+    register_target false
+  end
+end
diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb
index 612b404cb..c8241a1f1 100644
--- a/cookbooks/prometheus/recipes/server.rb
+++ b/cookbooks/prometheus/recipes/server.rb
@@ -150,6 +150,7 @@ search(:node, "roles:gateway") do |gateway|
 end
 
 jobs = {}
+snmp_targets = []
 
 search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client|
   if client[:prometheus][:mode] == "wireguard"
@@ -178,6 +179,15 @@ search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client
       :metric_relabel => metric_relabel
     }
   end
+
+  Hash(client[:prometheus][:snmp]).each do |instance, details|
+    snmp_targets << {
+      :instance => instance,
+      :target => details[:address],
+      :module => details[:module],
+      :address => client[:prometheus][:addresses]["snmp"]
+    }
+  end
 end
 
 certificates = search(:node, "letsencrypt:certificates").each_with_object({}) do |n, c|
@@ -218,7 +228,7 @@ template "/etc/prometheus/prometheus.yml" do
   owner "root"
   group "root"
   mode "644"
-  variables :jobs => jobs, :certificates => certificates
+  variables :jobs => jobs, :snmp_targets => snmp_targets, :certificates => certificates
 end
 
 template "/etc/prometheus/alert_rules.yml" do
diff --git a/cookbooks/prometheus/resources/exporter.rb b/cookbooks/prometheus/resources/exporter.rb
index 5eca43f5c..bab318a4a 100644
--- a/cookbooks/prometheus/resources/exporter.rb
+++ b/cookbooks/prometheus/resources/exporter.rb
@@ -59,6 +59,8 @@ action :create do
     only_if { node[:prometheus][:mode] == "external" }
   end
 
+  node.default[:prometheus][:addresses][new_resource.exporter] = listen_address
+
   if new_resource.register_target
     node.default[:prometheus][:exporters][new_resource.port] = {
       :name => new_resource.exporter,
diff --git a/cookbooks/prometheus/templates/default/prometheus.yml.erb b/cookbooks/prometheus/templates/default/prometheus.yml.erb
index 54f72a88c..812f32299 100644
--- a/cookbooks/prometheus/templates/default/prometheus.yml.erb
+++ b/cookbooks/prometheus/templates/default/prometheus.yml.erb
@@ -73,6 +73,28 @@ scrape_configs:
 <% end -%>
 <% end -%>
 <% end -%>
+  - job_name: snmp
+    scrape_interval: 5m
+    scrape_timeout: 1m
+    metrics_path: /snmp
+    static_configs:
+      - targets:
+<% @snmp_targets.sort_by { |t| t[:instance] }.each do |target| -%>
+      - targets:
+          - "<%= target[:target] %>/<%= target[:module] %>/<%= target[:address] %>"
+        labels:
+          instance: <%= target[:instance] %>
+<% end -%>
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: "([^/]+)/[^/]+/.*"
+        target_label: __param_target
+      - source_labels: [__address__]
+        regex: "[^/]+/([^/]+)/.*"
+        target_label: __param_module
+      - source_labels: [__address__]
+        regex: "[^/]+/[^/]+/(.*)"
+        target_label: __address__
 
 remote_write:
   - url: "http://localhost:9201/write"
diff --git a/roles/ironbelly.rb b/roles/ironbelly.rb
index 5042c4dd0..46bcde332 100644
--- a/roles/ironbelly.rb
+++ b/roles/ironbelly.rb
@@ -58,6 +58,13 @@ default_attributes(
   :planet => {
     :replication => "enabled"
   },
+  :prometheus => {
+    :snmp => {
+      "pdu1" => { :address => "10.0.48.100", :module => "apcups" },
+      "pdu2" => { :address => "10.0.48.101", :module => "apcups" },
+      "switch1" => { :address => "130.117.76.2", :module => "if_mib" }
+    }
+  },
   :rsyncd => {
     :modules => {
       :hosts => {