From a92c80c41d88faa20123f2841cc335c37905f638 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 24 Jul 2015 16:27:06 +0100 Subject: [PATCH] Add support for storing web logs in logstash --- cookbooks/apt/recipes/default.rb | 12 ++++ cookbooks/elasticsearch/attributes/default.rb | 1 - cookbooks/logstash/.foodcritic | 2 + cookbooks/logstash/README.md | 34 +++++++++ cookbooks/logstash/attributes/default.rb | 3 + .../logstash/files/default/lumberjack.crt | 19 +++++ cookbooks/logstash/metadata.rb | 8 +++ cookbooks/logstash/recipes/default.rb | 70 +++++++++++++++++++ cookbooks/logstash/recipes/forwarder.rb | 43 ++++++++++++ .../templates/default/logstash.conf.erb | 28 ++++++++ cookbooks/mediawiki/metadata.rb | 1 - cookbooks/mediawiki/recipes/default.rb | 1 - roles/elasticsearch.rb | 12 ++++ roles/foundation.rb | 1 + roles/ironbelly.rb | 1 + roles/logstash-forwarder.rb | 12 ++++ roles/logstash.rb | 13 ++++ roles/web-backend.rb | 9 +++ roles/web-frontend.rb | 9 +++ roles/wiki.rb | 1 + 20 files changed, 277 insertions(+), 3 deletions(-) delete mode 100644 cookbooks/elasticsearch/attributes/default.rb create mode 100644 cookbooks/logstash/.foodcritic create mode 100644 cookbooks/logstash/README.md create mode 100644 cookbooks/logstash/attributes/default.rb create mode 100644 cookbooks/logstash/files/default/lumberjack.crt create mode 100644 cookbooks/logstash/metadata.rb create mode 100644 cookbooks/logstash/recipes/default.rb create mode 100644 cookbooks/logstash/recipes/forwarder.rb create mode 100644 cookbooks/logstash/templates/default/logstash.conf.erb create mode 100644 roles/elasticsearch.rb create mode 100644 roles/logstash-forwarder.rb create mode 100644 roles/logstash.rb diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb index 050f806f6..0d793d4b7 100644 --- a/cookbooks/apt/recipes/default.rb +++ b/cookbooks/apt/recipes/default.rb @@ -87,6 +87,18 @@ apt_source "elasticsearch" do key "D88E42B4" end +apt_source "logstash" do + template "elasticsearch.list.erb" + url "http://packages.elasticsearch.org/logstash/1.5/debian" + key "D88E42B4" +end + +apt_source "logstash-forwarder" do + template "elasticsearch.list.erb" + url "http://packages.elasticsearch.org/logstashforwarder/debian" + key "D88E42B4" +end + apt_source "passenger" do url "https://oss-binaries.phusionpassenger.com/apt/passenger" key "AC40B2F7" diff --git a/cookbooks/elasticsearch/attributes/default.rb b/cookbooks/elasticsearch/attributes/default.rb deleted file mode 100644 index e5e5c3553..000000000 --- a/cookbooks/elasticsearch/attributes/default.rb +++ /dev/null @@ -1 +0,0 @@ -default[:apt][:sources] |= ["elasticsearch"] diff --git a/cookbooks/logstash/.foodcritic b/cookbooks/logstash/.foodcritic new file mode 100644 index 000000000..37c7131e2 --- /dev/null +++ b/cookbooks/logstash/.foodcritic @@ -0,0 +1,2 @@ +~FC001 +~FC003 diff --git a/cookbooks/logstash/README.md b/cookbooks/logstash/README.md new file mode 100644 index 000000000..205739199 --- /dev/null +++ b/cookbooks/logstash/README.md @@ -0,0 +1,34 @@ +Cookbook +======== +TODO: Enter the cookbook description here. + +e.g. +This cookbook makes your favorite breakfast sandwich. + +Requirements +------------ +TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. + +Attributes +---------- +TODO: List you cookbook attributes here. + +Usage +----- +TODO: Write usage instructions for each cookbook. + +Contributing +------------ +TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. + +e.g. +1. Fork the repository on Github +2. Create a named feature branch (like `add_component_x`) +3. Write your change +4. Write tests for your change (if applicable) +5. Run the tests, ensuring they all pass +6. Submit a Pull Request using Github + +License and Authors +------------------- +Authors: TODO: List authors diff --git a/cookbooks/logstash/attributes/default.rb b/cookbooks/logstash/attributes/default.rb new file mode 100644 index 000000000..9a56f3836 --- /dev/null +++ b/cookbooks/logstash/attributes/default.rb @@ -0,0 +1,3 @@ +default[:logstash][:forwarder][:network][:servers] = ["logstash.openstreetmap.org:5043"] +default[:logstash][:forwarder][:network][:"ssl ca"] = "/var/lib/logstash-forwarder/lumberjack.crt" +default[:logstash][:forwarder][:files] = [] diff --git a/cookbooks/logstash/files/default/lumberjack.crt b/cookbooks/logstash/files/default/lumberjack.crt new file mode 100644 index 000000000..d4d0ef4f7 --- /dev/null +++ b/cookbooks/logstash/files/default/lumberjack.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHTCCAgWgAwIBAgIJAKZiPjhmhjctMA0GCSqGSIb3DQEBCwUAMCUxIzAhBgNV +BAMMGmxvZ3N0YXNoLm9wZW5zdHJlZXRtYXAub3JnMB4XDTE1MDcyMzIzMDkxNVoX +DTE1MDgyMjIzMDkxNVowJTEjMCEGA1UEAwwabG9nc3Rhc2gub3BlbnN0cmVldG1h +cC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQClqGmqXzf4Woj3 +Y3JEVaOdSWCKODABCMsbCE5Wca3+72xQwtwI/jhH+IMdXk2+4Puw+5j01Ko3dGbu +ZWPZ8vWfVg0QXMygZTLNj/Vd+vKZ77DDiaeJHtYudQ0Q5X1sVDScNHeE20IwGnot +5QcnvzxRhFPaP/UFP4x6v0UUMWVbI211mD+Ebdx6o0829hI+NYNmbnCmk8gBIBQ7 +YaddCF8MDAJDjFIcC/+DrxPF8iFa2lEcpGcsldVBVcgFaNVA/cXI/ysncZjJGigv +mnk8Oq6xahfdBZyae53XbaO3AghHtJ9kMOYPLLmcMDWJcuFHQOS3NH4HeZ5O5chk +xsLahqm3AgMBAAGjUDBOMB0GA1UdDgQWBBTiloMwMnJ4n+8f4qLqKW2Ee4FyfjAf +BgNVHSMEGDAWgBTiloMwMnJ4n+8f4qLqKW2Ee4FyfjAMBgNVHRMEBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4IBAQB/KFOEA9VvqA/85C0VDAu/3kPXUnCkSNW8UU7xGJTY +ac5tN8EP9hA+sUmqxTyqt5HlFVPFnM+d/qMbhS5cKttt60F0deDMEwwncQoT+tcC +oRQPLQshkCUCWA/khAAeYvD9BsFM1jVan8HultTksLZ3U0DwDkn47K4CCVhssv0W +z2aJctJQbvkADjeQ5eVgatrHpHv8GcXyN8olPbDHo9IH+7nw1lx1oaAIYuc9oHhW +4ZMF5hBCTgOjq48O7604V731hxVC3lzYaT3qURPW9fOu9qzDSza4SUaJvATpfLrj +sHJ7QDlOSt11AjHDD6C01L5SeMturwRe4lWYzCiNuW4D +-----END CERTIFICATE----- diff --git a/cookbooks/logstash/metadata.rb b/cookbooks/logstash/metadata.rb new file mode 100644 index 000000000..5e4405a21 --- /dev/null +++ b/cookbooks/logstash/metadata.rb @@ -0,0 +1,8 @@ +name "logstash" +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache 2.0" +description "Installs and configures a elasticsearch server" +long_description IO.read(File.join(File.dirname(__FILE__), "README.md")) +version "1.0.0" +depends "networking" diff --git a/cookbooks/logstash/recipes/default.rb b/cookbooks/logstash/recipes/default.rb new file mode 100644 index 000000000..b2932aa97 --- /dev/null +++ b/cookbooks/logstash/recipes/default.rb @@ -0,0 +1,70 @@ +# +# Cookbook Name:: logstash +# Recipe:: default +# +# Copyright 2015, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "networking" + +keys = data_bag_item("logstash", "keys") + +package "openjdk-7-jre-headless" +package "logstash" + +cookbook_file "/var/lib/logstash/lumberjack.crt" do + source "lumberjack.crt" + user "root" + group "logstash" + mode 0644 + notifies :restart, "service[logstash]" +end + +file "/var/lib/logstash/lumberjack.key" do + content keys["lumberjack"].join("\n") + user "root" + group "logstash" + mode 0640 + notifies :restart, "service[logstash]" +end + +template "/etc/logstash/conf.d/chef.conf" do + source "logstash.conf.erb" + user "root" + group "root" + mode 0644 + notifies :restart, "service[logstash]" +end + +service "logstash" do + action [:enable, :start] + supports :status => true, :restart => true +end + +forwarders = search(:node, "recipes:logstash\\:\\:forwarder") + +forwarders.each do |forwarder| + forwarder.interfaces(:role => :external) do |interface| + firewall_rule "accept-lumberjack-#{forwarder}" do + action :accept + family interface[:family] + source "#{interface[:zone]}:#{interface[:address]}" + dest "fw" + proto "tcp:syn" + dest_ports "5043" + source_ports "1024:" + end + end +end diff --git a/cookbooks/logstash/recipes/forwarder.rb b/cookbooks/logstash/recipes/forwarder.rb new file mode 100644 index 000000000..093912d7c --- /dev/null +++ b/cookbooks/logstash/recipes/forwarder.rb @@ -0,0 +1,43 @@ +# +# Cookbook Name:: logstash +# Recipe:: default +# +# Copyright 2015, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "json" + +package "logstash-forwarder" + +cookbook_file "/var/lib/logstash-forwarder/lumberjack.crt" do + source "lumberjack.crt" + user "root" + group "root" + mode 0644 + notifies :restart, "service[logstash-forwarder]" +end + +file "/etc/logstash-forwarder.conf" do + content JSON.pretty_generate(node[:logstash][:forwarder]) + user "root" + group "root" + mode 0644 + notifies :restart, "service[logstash-forwarder]" +end + +service "logstash-forwarder" do + action [:enable, :start] + supports :status => true, :restart => true +end diff --git a/cookbooks/logstash/templates/default/logstash.conf.erb b/cookbooks/logstash/templates/default/logstash.conf.erb new file mode 100644 index 000000000..f6c207a61 --- /dev/null +++ b/cookbooks/logstash/templates/default/logstash.conf.erb @@ -0,0 +1,28 @@ +input { + lumberjack { + port => 5043 + ssl_certificate => "/var/lib/logstash/lumberjack.crt" + ssl_key => "/var/lib/logstash/lumberjack.key" + } +} + +filter { + if [type] == "apache" { + grok { + match => [ "message", "%{COMBINEDAPACHELOG} %{NUMBER:duration:int}us %{WORD:request_id} %{NOTSPACE:ssl_protocol} %{NOTSPACE:ssl_cipher}" ] + } + date { + match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] + } + } else if [type] == "rails" { + json { + source => "message" + } + } +} + +output { + elasticsearch { + host => [ "127.0.0.1" ] + } +} diff --git a/cookbooks/mediawiki/metadata.rb b/cookbooks/mediawiki/metadata.rb index 2551b291e..46daf9a65 100644 --- a/cookbooks/mediawiki/metadata.rb +++ b/cookbooks/mediawiki/metadata.rb @@ -9,4 +9,3 @@ depends "memcached" depends "apache" depends "mysql" depends "git" -depends "elasticsearch" diff --git a/cookbooks/mediawiki/recipes/default.rb b/cookbooks/mediawiki/recipes/default.rb index 61c761725..267c4f4b1 100644 --- a/cookbooks/mediawiki/recipes/default.rb +++ b/cookbooks/mediawiki/recipes/default.rb @@ -21,7 +21,6 @@ include_recipe "memcached" include_recipe "apache::ssl" include_recipe "mysql" include_recipe "git" -include_recipe "elasticsearch" # Mediawiki Base Requirements package "php5" diff --git a/roles/elasticsearch.rb b/roles/elasticsearch.rb new file mode 100644 index 000000000..ae0e2b6cc --- /dev/null +++ b/roles/elasticsearch.rb @@ -0,0 +1,12 @@ +name "elasticsearch" +description "Role applied to all elasticsearch servers" + +default_attributes( + :apt => { + :sources => ["elasticsearch"] + } +) + +run_list( + "recipe[elasticsearch]" +) diff --git a/roles/foundation.rb b/roles/foundation.rb index fa96196f5..031611775 100644 --- a/roles/foundation.rb +++ b/roles/foundation.rb @@ -19,6 +19,7 @@ default_attributes( run_list( "role[crm]", + "role[elasticsearch]", "recipe[foundation::wiki]", "recipe[foundation::board]" ) diff --git a/roles/ironbelly.rb b/roles/ironbelly.rb index 4bcb8fb26..fb86c78f0 100644 --- a/roles/ironbelly.rb +++ b/roles/ironbelly.rb @@ -97,6 +97,7 @@ run_list( "role[stats]", "role[planet]", "role[planetdump]", + "role[logstash]", "recipe[rsyncd]", "recipe[openvpn]", "recipe[git::server]", diff --git a/roles/logstash-forwarder.rb b/roles/logstash-forwarder.rb new file mode 100644 index 000000000..2857223b1 --- /dev/null +++ b/roles/logstash-forwarder.rb @@ -0,0 +1,12 @@ +name "logstash-forwarder" +description "Role applied to all logstash forwarders" + +default_attributes( + :apt => { + :sources => ["logstash-forwarder"] + } +) + +run_list( + "recipe[logstash::forwarder]" +) diff --git a/roles/logstash.rb b/roles/logstash.rb new file mode 100644 index 000000000..dabe91379 --- /dev/null +++ b/roles/logstash.rb @@ -0,0 +1,13 @@ +name "logstash" +description "Role applied to all logstash servers" + +default_attributes( + :apt => { + :sources => ["logstash"] + } +) + +run_list( + "role[elasticsearch]", + "recipe[logstash]" +) diff --git a/roles/web-backend.rb b/roles/web-backend.rb index d77ea1b95..51d7c477e 100644 --- a/roles/web-backend.rb +++ b/roles/web-backend.rb @@ -8,6 +8,14 @@ default_attributes( :max_requests_per_child => 10000 } }, + :logstash => { + :forwarder => { + :files => [ + { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } }, + { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } } + ] + } + }, :memcached => { :memory_limit => 4096 }, @@ -18,5 +26,6 @@ default_attributes( run_list( "role[web]", + "role[logstash-forwarder]", "recipe[web::backend]" ) diff --git a/roles/web-frontend.rb b/roles/web-frontend.rb index faa1017de..e228e0efc 100644 --- a/roles/web-frontend.rb +++ b/roles/web-frontend.rb @@ -13,6 +13,14 @@ default_attributes( :max_requests_per_child => 10000 } }, + :logstash => { + :forwarder => { + :files => [ + { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } }, + { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } } + ] + } + }, :passenger => { :max_pool_size => 50 }, @@ -34,5 +42,6 @@ default_attributes( run_list( "role[web]", + "role[logstash-forwarder]", "recipe[web::frontend]" ) diff --git a/roles/wiki.rb b/roles/wiki.rb index e79639928..4357ee17a 100644 --- a/roles/wiki.rb +++ b/roles/wiki.rb @@ -39,5 +39,6 @@ default_attributes( ) run_list( + "role[elasticsearch]", "recipe[wiki]" ) -- 2.39.5