From ae56a522a5782d71b7b758464e2592ac0d2ef07f Mon Sep 17 00:00:00 2001
From: Tom Hughes <tom@compton.nu>
Date: Thu, 5 Jul 2018 19:19:49 +0100
Subject: [PATCH] Switch to use filebeat for forwarding to logstash

---
 cookbooks/apt/recipes/default.rb         |  6 -----
 cookbooks/logstash/.foodcritic           |  1 +
 cookbooks/logstash/attributes/default.rb |  7 ++---
 cookbooks/logstash/recipes/forwarder.rb  | 34 ++++++++++++++++--------
 roles/logstash-forwarder.rb              |  2 +-
 roles/web-backend.rb                     |  6 ++---
 roles/web-frontend.rb                    |  6 ++---
 7 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb
index 8f57b88c9..51586d61b 100644
--- a/cookbooks/apt/recipes/default.rb
+++ b/cookbooks/apt/recipes/default.rb
@@ -138,12 +138,6 @@ apt_source "logstash" do
   key "D27D666CD88E42B4"
 end
 
-apt_source "logstash-forwarder" do
-  source_template "elasticsearch.list.erb"
-  url "https://packages.elasticsearch.org/logstashforwarder/debian"
-  key "D27D666CD88E42B4"
-end
-
 apt_source "passenger" do
   url "https://oss-binaries.phusionpassenger.com/apt/passenger"
   key "561F9B9CAC40B2F7"
diff --git a/cookbooks/logstash/.foodcritic b/cookbooks/logstash/.foodcritic
index 53c271a0f..563333674 100644
--- a/cookbooks/logstash/.foodcritic
+++ b/cookbooks/logstash/.foodcritic
@@ -1,5 +1,6 @@
 ~FC001
 ~FC003
+~FC019
 ~FC064
 ~FC065
 ~FC066
diff --git a/cookbooks/logstash/attributes/default.rb b/cookbooks/logstash/attributes/default.rb
index 9a56f3836..4014b7207 100644
--- a/cookbooks/logstash/attributes/default.rb
+++ b/cookbooks/logstash/attributes/default.rb
@@ -1,3 +1,4 @@
-default[:logstash][:forwarder][:network][:servers] = ["logstash.openstreetmap.org:5043"]
-default[:logstash][:forwarder][:network][:"ssl ca"] = "/var/lib/logstash-forwarder/lumberjack.crt"
-default[:logstash][:forwarder][:files] = []
+default[:logstash][:forwarder]["output.logstash"]["hosts"] = ["logstash.openstreetmap.org:5044"]
+default[:logstash][:forwarder]["output.logstash"]["ssl.certificate_authorities"] = "/etc/filebeat/filebeat.crt"
+default[:logstash][:forwarder]["output.logstash"]["ssl.verification_mode"] = "none"
+default[:logstash][:forwarder]["filebeat.prospectors"] = []
diff --git a/cookbooks/logstash/recipes/forwarder.rb b/cookbooks/logstash/recipes/forwarder.rb
index d6956a8f7..ea653c8ef 100644
--- a/cookbooks/logstash/recipes/forwarder.rb
+++ b/cookbooks/logstash/recipes/forwarder.rb
@@ -17,27 +17,39 @@
 # limitations under the License.
 #
 
-require "json"
+service "logstash-forwarder" do
+  action [:disable, :stop]
+end
+
+file "/var/lib/logstash-forwarder/lumberjack.crt" do
+  action :delete
+end
 
-package "logstash-forwarder"
+package "logstash-forwarder" do
+  action :purge
+end
+
+require "yaml"
 
-cookbook_file "/var/lib/logstash-forwarder/lumberjack.crt" do
-  source "lumberjack.crt"
+package "filebeat"
+
+cookbook_file "/etc/filebeat/filebeat.crt" do
+  source "beats.crt"
   user "root"
   group "root"
-  mode 0o644
-  notifies :restart, "service[logstash-forwarder]"
+  mode 0o600
+  notifies :restart, "service[filebeat]"
 end
 
-file "/etc/logstash-forwarder.conf" do
-  content JSON.pretty_generate(node[:logstash][:forwarder])
+file "/etc/filebeat/filebeat.yml" do
+  content YAML.dump(node[:logstash][:forwarder].to_hash)
   user "root"
   group "root"
-  mode 0o644
-  notifies :restart, "service[logstash-forwarder]"
+  mode 0o600
+  notifies :restart, "service[filebeat]"
 end
 
-service "logstash-forwarder" do
+service "filebeat" do
   action [:enable, :start]
   supports :status => true, :restart => true
 end
diff --git a/roles/logstash-forwarder.rb b/roles/logstash-forwarder.rb
index 3d8ba3abe..60b34a311 100644
--- a/roles/logstash-forwarder.rb
+++ b/roles/logstash-forwarder.rb
@@ -3,7 +3,7 @@ description "Role applied to all logstash forwarders"
 
 default_attributes(
   :apt => {
-    # :sources => ["logstash-forwarder"]
+    :sources => ["elasticsearch5.x"]
   }
 )
 
diff --git a/roles/web-backend.rb b/roles/web-backend.rb
index 4c6dff43f..d59ce812e 100644
--- a/roles/web-backend.rb
+++ b/roles/web-backend.rb
@@ -10,9 +10,9 @@ default_attributes(
   },
   :logstash => {
     :forwarder => {
-      :files => [
-        { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
-        { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+      "filebeat.prospectors" => [
+        { "input_type" => "log", "paths" => ["/var/log/apache2/access.log"], "fields" => { "type" => "apache" } },
+        { "input_type" => "log", "paths" => ["/var/log/web/rails-logstash.log"], "fields" => { "type" => "rails" } }
       ]
     }
   },
diff --git a/roles/web-frontend.rb b/roles/web-frontend.rb
index 4d82bd4f6..5f23e0618 100644
--- a/roles/web-frontend.rb
+++ b/roles/web-frontend.rb
@@ -15,9 +15,9 @@ default_attributes(
   },
   :logstash => {
     :forwarder => {
-      :files => [
-        { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
-        { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+      "filebeat.prospectors" => [
+        { "input_type" => "log", "paths" => ["/var/log/apache2/access.log"], "fields" => { "type" => "apache" } },
+        { "input_type" => "log", "paths" => ["/var/log/web/rails-logstash.log"], "fields" => { "type" => "rails" } }
       ]
     }
   },
-- 
2.39.5