X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/fdd1bdfdaf34a4915e79aad85ad87c0f783e425a..0f8970107f610c90fd9a8377eebb065893569a27:/cookbooks/hardware/recipes/default.rb diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 92abdf65d..b1d7caf0b 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -1,14 +1,14 @@ # -# Cookbook Name:: hardware +# Cookbook:: hardware # Recipe:: default # -# Copyright 2012, OpenStreetMap Foundation +# Copyright:: 2012, OpenStreetMap Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -17,16 +17,22 @@ # limitations under the License. # -case node[:cpu][:"0"][:vendor_id] -when "GenuineIntel" - package "intel-microcode" +include_recipe "apt" +include_recipe "git" +include_recipe "munin" +include_recipe "prometheus" +include_recipe "sysfs" +include_recipe "tools" + +ohai_plugin "hardware" do + template "ohai.rb.erb" end case node[:cpu][:"0"][:vendor_id] +when "GenuineIntel" + package "intel-microcode" when "AuthenticAMD" - if node[:lsb][:release].to_f >= 14.04 - package "amd64-microcode" - end + package "amd64-microcode" end if node[:dmi] && node[:dmi][:system] @@ -43,92 +49,107 @@ else product = "Unknown" end +units = [] + +if node[:roles].include?("bytemark") || node[:roles].include?("exonetric") || node[:roles].include?("prgmr") + units << "0" +end + case manufacturer when "HP" package "hponcfg" - package "hp-health" - unit = "1" - speed = "115200" -when "TYAN" - unit = "0" - speed = "115200" -when "TYAN Computer Corporation" - unit = "0" - speed = "115200" -when "Supermicro" - case product - when "H8DGU", "X9SCD", "X7DBU", "X7DW3", "X9DR7/E-(J)LN4F", "X9DR3-F", "X9DRW" - unit = "1" - speed = "115200" - else - unit = "0" - speed = "115200" - end -when "IBM" - unit = "0" - speed = "115200" -end - -if manufacturer == "HP" && node[:lsb][:release].to_f > 11.10 - include_recipe "git" - git "/opt/hp/hp-legacy" do - action :sync - repository "git://chef.openstreetmap.org/hp-legacy.git" - user "root" - group "root" + package "hp-health" do + action :install + notifies :restart, "service[hp-health]" end - link "/opt/hp/hp-health/bin/hpasmd" do - to "/opt/hp/hp-legacy/hpasmd" + service "hp-health" do + action [:enable, :start] + supports :status => true, :restart => true end - link "/usr/lib/libhpasmintrfc.so.3.0" do - to "/opt/hp/hp-legacy/libhpasmintrfc.so.3.0" - end + if product.end_with?("Gen8", "Gen9") + package "hp-ams" do + action :install + notifies :restart, "service[hp-ams]" + end - link "/usr/lib/libhpasmintrfc.so.3" do - to "libhpasmintrfc.so.3.0" + service "hp-ams" do + action [:enable, :start] + supports :status => true, :restart => true + end end - link "/usr/lib/libhpasmintrfc.so" do - to "libhpasmintrfc.so.3.0" - end -end + units << "1" +when "TYAN" + units << "0" +when "TYAN Computer Corporation" + units << "0" +when "Supermicro" + units << "1" +when "IBM" + units << "0" +when "VMware, Inc." + package "open-vm-tools" -unless unit.nil? - file "/etc/init/ttySttyS#{unit}.conf" do + # Remove timeSync plugin completely + # https://github.com/vmware/open-vm-tools/issues/302 + file "/usr/lib/open-vm-tools/plugins/vmsvc/libtimeSync.so" do action :delete + notifies :restart, "service[open-vm-tools]" end - template "/etc/init/ttyS#{unit}.conf" do - source "tty.conf.erb" - owner "root" - group "root" - mode 0644 - variables :unit => unit, :speed => speed + # Attempt to tell Host we are not interested in timeSync + execute "vmware-toolbox-cmd-timesync-disable" do + command "/usr/bin/vmware-toolbox-cmd timesync disable" + ignore_failure true end - service "ttyS#{unit}" do - provider Chef::Provider::Service::Upstart + service "open-vm-tools" do action [:enable, :start] - supports :status => true, :restart => true, :reload => false - subscribes :restart, "template[/etc/init/ttyS#{unit}.conf]" + supports :status => true, :restart => true end end +units.sort.uniq.each do |unit| + service "serial-getty@ttyS#{unit}" do + action [:enable, :start] + end +end + +# if we need a different / special kernel version to make the hardware +# work (e.g: https://github.com/openstreetmap/operations/issues/45) then +# ensure that we have the package installed. the grub template will +# make sure that this is the default on boot. +if node[:hardware][:grub][:kernel] + kernel_version = node[:hardware][:grub][:kernel] + + package "linux-image-#{kernel_version}-generic" + package "linux-image-extra-#{kernel_version}-generic" + package "linux-headers-#{kernel_version}-generic" + package "linux-tools-#{kernel_version}-generic" + + boot_device = IO.popen(["df", "/boot"]).readlines.last.split.first + boot_uuid = IO.popen(["blkid", "-o", "value", "-s", "UUID", boot_device]).readlines.first.chomp + grub_entry = "gnulinux-advanced-#{boot_uuid}>gnulinux-#{kernel_version}-advanced-#{boot_uuid}" +else + grub_entry = "0" +end + if File.exist?("/etc/default/grub") execute "update-grub" do action :nothing command "/usr/sbin/update-grub" + not_if { ENV["TEST_KITCHEN"] } end template "/etc/default/grub" do source "grub.erb" owner "root" group "root" - mode 0644 - variables :unit => unit, :speed => speed + mode "644" + variables :units => units, :entry => grub_entry notifies :run, "execute[update-grub]" end end @@ -144,7 +165,7 @@ template "/etc/initramfs-tools/conf.d/mdadm" do source "initramfs-mdadm.erb" owner "root" group "root" - mode 0644 + mode "644" notifies :run, "execute[update-initramfs]" end @@ -155,67 +176,77 @@ end if node[:kernel][:modules].include?("ipmi_si") package "ipmitool" + package "freeipmi-tools" + + prometheus_exporter "ipmi" do + port 9290 + end end -if node[:lsb][:release].to_f >= 12.10 - package "irqbalance" +package "irqbalance" - template "/etc/default/irqbalance" do - source "irqbalance.erb" - owner "root" - group "root" - mode 0644 - end +service "irqbalance" do + action [:start, :enable] + supports :status => false, :restart => true, :reload => false +end - service "irqbalance" do - action [:start, :enable] - supports :status => false, :restart => true, :reload => false - subscribes :restart, "template[/etc/default/irqbalance]" - end +# Link Layer Discovery Protocol Daemon +package "lldpd" +service "lldpd" do + action [:start, :enable] + supports :status => true, :restart => true, :reload => true end tools_packages = [] status_packages = {} -node[:kernel][:modules].each_key do |modname| - case modname - when "cciss" - tools_packages << "hpacucli" - status_packages["cciss-vol-status"] ||= [] - when "hpsa" - tools_packages << "hpacucli" - status_packages["cciss-vol-status"] ||= [] - when "mptsas" - tools_packages << "lsiutil" - # status_packages["mpt-status"] ||= [] - when "mpt2sas" - tools_packages << "sas2ircu" - status_packages["sas2ircu-status"] ||= [] - when "megaraid_mm" - tools_packages << "megactl" - status_packages["megaraid-status"] ||= [] - when "megaraid_sas" - tools_packages << "megacli" - status_packages["megaclisas-status"] ||= [] - when "aacraid" - tools_packages << "arcconf" - status_packages["aacraid-status"] ||= [] +if node[:virtualization][:role] != "guest" || + (node[:virtualization][:system] != "lxc" && + node[:virtualization][:system] != "lxd" && + node[:virtualization][:system] != "openvz") + + node[:kernel][:modules].each_key do |modname| + case modname + when "cciss" + tools_packages << "ssacli" + status_packages["cciss-vol-status"] ||= [] + when "hpsa" + tools_packages << "ssacli" + status_packages["cciss-vol-status"] ||= [] + when "mptsas" + tools_packages << "lsiutil" + status_packages["mpt-status"] ||= [] + when "mpt2sas", "mpt3sas" + tools_packages << "sas2ircu" + status_packages["sas2ircu-status"] ||= [] + when "megaraid_mm" + tools_packages << "megactl" + status_packages["megaraid-status"] ||= [] + when "megaraid_sas" + tools_packages << "megacli" + status_packages["megaclisas-status"] ||= [] + when "aacraid" + tools_packages << "arcconf" + status_packages["aacraid-status"] ||= [] + when "arcmsr" + tools_packages << "areca" + end end -end -node[:block_device].each do |name, attributes| - next unless attributes[:vendor] == "HP" && attributes[:model] == "LOGICAL VOLUME" + node[:block_device].each do |name, attributes| + next unless attributes[:vendor] == "HP" && attributes[:model] == "LOGICAL VOLUME" - if name =~ /^cciss!(c[0-9]+)d[0-9]+$/ - status_packages["cciss-vol-status"] |= ["cciss/#{Regexp.last_match[1]}d0"] - else - Dir.glob("/sys/block/#{name}/device/scsi_generic/*").each do |sg| - status_packages["cciss-vol-status"] |= [File.basename(sg)] + if name =~ /^cciss!(c[0-9]+)d[0-9]+$/ + status_packages["cciss-vol-status"] |= ["cciss/#{Regexp.last_match[1]}d0"] + else + Dir.glob("/sys/block/#{name}/device/scsi_generic/*").each do |sg| + status_packages["cciss-vol-status"] |= [File.basename(sg)] + end end end end -%w(hpacucli lsiutil sas2ircu megactl megacli arcconf).each do |tools_package| +%w[ssacli lsiutil sas2ircu megactl megacli arcconf].each do |tools_package| if tools_packages.include?(tools_package) package tools_package else @@ -225,7 +256,53 @@ end end end -["cciss-vol-status", "mpt-status", "sas2ircu-status", "megaraid-status", "megaclisas-status", "aacraid-status"].each do |status_package| +if tools_packages.include?("areca") + include_recipe "git" + + git "/opt/areca" do + action :sync + repository "https://git.openstreetmap.org/private/areca.git" + depth 1 + user "root" + group "root" + not_if { ENV["TEST_KITCHEN"] } + end +else + directory "/opt/areca" do + action :delete + recursive true + end +end + +if status_packages.include?("cciss-vol-status") + template "/usr/local/bin/cciss-vol-statusd" do + source "cciss-vol-statusd.erb" + owner "root" + group "root" + mode "755" + notifies :restart, "service[cciss-vol-statusd]" + end + + systemd_service "cciss-vol-statusd" do + description "Check cciss_vol_status values in the background" + exec_start "/usr/local/bin/cciss-vol-statusd" + private_tmp true + protect_system "full" + protect_home true + no_new_privileges true + notifies :restart, "service[cciss-vol-statusd]" + end +else + systemd_service "cciss-vol-statusd" do + action :delete + end + + template "/usr/local/bin/cciss-vol-statusd" do + action :delete + end +end + +%w[cciss-vol-status mpt-status sas2ircu-status megaraid-status megaclisas-status aacraid-status].each do |status_package| if status_packages.include?(status_package) package status_package @@ -233,7 +310,7 @@ end source "raid.default.erb" owner "root" group "root" - mode 0644 + mode "644" variables :devices => status_packages[status_package] end @@ -253,98 +330,187 @@ end end end -disks = [] +disks = if node[:hardware][:disk] + node[:hardware][:disk][:disks] + else + [] + end -node[:block_device].each do |name, attributes| - disks << { :device => name } if attributes[:vendor] == "ATA" -end +intel_ssds = disks.select { |d| d[:vendor] == "INTEL" && d[:model] =~ /^SSD/ } -if status_packages["cciss-vol-status"] - status_packages["cciss-vol-status"].each do |device| - IO.popen(["cciss_vol_status", "-V", "/dev/#{device}"]).each do |line| - disks << { :device => device, :driver => "cciss", :id => Regexp.last_match[1].to_i - 1 } if line =~ / bay ([0-9]+) +HP / - end - end -end +nvmes = if node[:hardware][:pci] + node[:hardware][:pci].values.select { |pci| pci[:driver] == "nvme" } + else + [] + end -if status_packages["megaclisas-status"] - controller = 0 +intel_nvmes = nvmes.select { |pci| pci[:vendor_name] == "Intel Corporation" } - Dir.glob("/sys/class/scsi_host/host*") do |host| - driver = File.new("#{host}/proc_name").read.chomp +if !intel_ssds.empty? || !intel_nvmes.empty? + package "unzip" - next unless driver == "megaraid_sas" + intel_mas_tool_version = "1.4" + intel_mas_package_version = "#{intel_mas_tool_version}.102-0" - bus = host.sub("/sys/class/scsi_host/host", "") - device = File.basename(Dir.glob("/sys/bus/scsi/devices/#{bus}:*/scsi_generic/*").first) + remote_file "#{Chef::Config[:file_cache_path]}/Intel_MAS_CLI_Tool_#{intel_mas_tool_version}_Linux.zip" do + source "https://downloadmirror.intel.com/30059/eng/Intel%C2%AE_MAS_CLI_Tool_Linux.zip" + end - IO.popen(["megacli", "-PDList", "-a#{controller}", "-NoLog"]).each do |line| - disks << { :device => device, :driver => "megaraid", :id => Regexp.last_match[1] } if line =~ /^Device Id: ([0-9]+)$/ + execute "#{Chef::Config[:file_cache_path]}/Intel_MAS_CLI_Tool_#{intel_mas_tool_version}_Linux.zip" do + command "unzip Intel_MAS_CLI_Tool_#{intel_mas_tool_version}_Linux.zip intelmas_#{intel_mas_package_version}_amd64.deb" + cwd Chef::Config[:file_cache_path] + user "root" + group "root" + not_if { ::File.exist?("#{Chef::Config[:file_cache_path]}/intelmas_#{intel_mas_package_version}_amd64.deb") } + end - disks.pop if line =~ /^Firmware state: Hotspare, Spun down$/ - end + dpkg_package "intelmas" do + version "#{intel_mas_package_version}" + source "#{Chef::Config[:file_cache_path]}/intelmas_#{intel_mas_package_version}_amd64.deb" + end - controller += 1 + dpkg_package "isdct" do + action :purge end end -if status_packages["sas2ircu-status"] - Dir.glob("/sys/class/scsi_host/host*") do |host| - driver = File.new("#{host}/proc_name").read.chomp - - next unless driver == "mpt2sas" - - bus = host.sub("/sys/class/scsi_host/host", "") - - Dir.glob("/sys/bus/scsi/devices/#{bus}:0:*/scsi_generic/*").each do |sg| - disks << { :device => File.basename(sg) } +disks = disks.map do |disk| + next if disk[:state] == "spun_down" || %w[unconfigured failed].any?(disk[:status]) + + if disk[:smart_device] + controller = node[:hardware][:disk][:controllers][disk[:controller]] + + if controller && controller[:device] + device = controller[:device].sub("/dev/", "") + smart = disk[:smart_device] + + if device.start_with?("cciss/") && smart =~ /^cciss,(\d+)$/ + array = node[:hardware][:disk][:arrays][disk[:arrays].first] + munin = "cciss-3#{array[:wwn]}-#{Regexp.last_match(1)}" + elsif smart =~ /^.*,(\d+)$/ + munin = "#{device}-#{Regexp.last_match(1)}" + elsif smart =~ %r{^.*,(\d+)/(\d+)$} + munin = "#{device}-#{Regexp.last_match(1)}:#{Regexp.last_match(2)}" + end + elsif disk[:device] + device = disk[:device].sub("/dev/", "") + smart = disk[:smart_device] + + if smart =~ /^.*,(\d+),(\d+),(\d+)$/ + munin = "#{device}-#{Regexp.last_match(1)}:#{Regexp.last_match(2)}:#{Regexp.last_match(3)}" + end end - end + elsif disk[:device] =~ %r{^/dev/(nvme\d+)n\d+$} + device = Regexp.last_match(1) + munin = device + elsif disk[:device] + device = disk[:device].sub("/dev/", "") + munin = device + end + + next if device.nil? + + Hash[ + :device => device, + :smart => smart, + :munin => munin, + :hddtemp => munin.tr("-:", "_") + ] end -if status_packages["aacraid-status"] - Dir.glob("/sys/class/scsi_host/host*") do |host| - driver = File.new("#{host}/proc_name").read.chomp +disks = disks.compact.uniq - next unless driver == "aacraid" - - bus = host.sub("/sys/class/scsi_host/host", "") +if disks.count.positive? + package "smartmontools" - Dir.glob("/sys/bus/scsi/devices/#{bus}:1:*/scsi_generic/*").each do |sg| - disks << { :device => File.basename(sg) } - end + template "/etc/cron.daily/update-smart-drivedb" do + source "update-smart-drivedb.erb" + owner "root" + group "root" + mode "755" end -end -if disks.count > 0 - package "smartmontools" + template "/usr/local/bin/smartd-mailer" do + source "smartd-mailer.erb" + owner "root" + group "root" + mode "755" + end template "/etc/smartd.conf" do source "smartd.conf.erb" owner "root" group "root" - mode 0644 + mode "644" variables :disks => disks - notifies :reload, "service[smartmontools]" end template "/etc/default/smartmontools" do source "smartmontools.erb" owner "root" group "root" - mode 0644 - notifies :restart, "service[smartmontools]" + mode "644" end service "smartmontools" do action [:enable, :start] + subscribes :reload, "template[/etc/smartd.conf]" + subscribes :restart, "template[/etc/default/smartmontools]" + end + + template "/etc/prometheus/collectors/smart.devices" do + source "smart.devices.erb" + owner "root" + group "root" + mode "644" + variables :disks => disks + end + + prometheus_collector "smart" do + interval "15m" + end + + # Don't try and do munin monitoring of disks behind + # an Areca controller as they only allow one thing to + # talk to the controller at a time and smartd will + # throw errors if it clashes with munin + disks = disks.reject { |disk| disk[:smart]&.start_with?("areca,") } + + disks.each do |disk| + munin_plugin "smart_#{disk[:munin]}" do + target "smart_" + conf "munin.smart.erb" + conf_variables :disk => disk + end end else - service "smartmontools" do + service "smartd" do action [:stop, :disable] end end +if disks.count.positive? + munin_plugin "hddtemp_smartctl" do + conf "munin.hddtemp.erb" + conf_variables :disks => disks + end +else + munin_plugin "hddtemp_smartctl" do + action :delete + conf "munin.hddtemp.erb" + end +end + +plugins = Dir.glob("/etc/munin/plugins/smart_*").map { |p| File.basename(p) } - + disks.map { |d| "smart_#{d[:munin]}" } + +plugins.each do |plugin| + munin_plugin plugin do + action :delete + conf "munin.smart.erb" + end +end + if File.exist?("/etc/mdadm/mdadm.conf") mdadm_conf = edit_file "/etc/mdadm/mdadm.conf" do |line| line.gsub!(/^MAILADDR .*$/, "MAILADDR admins@openstreetmap.org") @@ -355,7 +521,7 @@ if File.exist?("/etc/mdadm/mdadm.conf") file "/etc/mdadm/mdadm.conf" do owner "root" group "root" - mode 0644 + mode "644" content mdadm_conf end @@ -369,21 +535,12 @@ template "/etc/modules" do source "modules.erb" owner "root" group "root" - mode 0644 + mode "644" end -if node[:lsb][:release].to_f <= 12.10 - service "module-init-tools" do - provider Chef::Provider::Service::Upstart - action :nothing - subscribes :start, "template[/etc/modules]" - end -else - service "kmod" do - provider Chef::Provider::Service::Upstart - action :nothing - subscribes :start, "template[/etc/modules]" - end +service "kmod" do + action :nothing + subscribes :start, "template[/etc/modules]" end if node[:hardware][:watchdog] @@ -393,7 +550,7 @@ if node[:hardware][:watchdog] source "watchdog.erb" owner "root" group "root" - mode 0644 + mode "644" variables :module => node[:hardware][:watchdog] end @@ -405,6 +562,30 @@ end unless Dir.glob("/sys/class/hwmon/hwmon*").empty? package "lm-sensors" + Dir.glob("/sys/devices/platform/coretemp.*").each do |coretemp| + cpu = File.basename(coretemp).sub("coretemp.", "").to_i + chip = format("coretemp-isa-%04d", cpu) + + temps = if File.exist?("#{coretemp}/name") + Dir.glob("#{coretemp}/temp*_input").map do |temp| + File.basename(temp).sub("temp", "").sub("_input", "").to_i + end.sort + else + Dir.glob("#{coretemp}/hwmon/hwmon*/temp*_input").map do |temp| + File.basename(temp).sub("temp", "").sub("_input", "").to_i + end.sort + end + + if temps.first == 1 + node.default[:hardware][:sensors][chip][:temps][:temp1][:label] = "CPU #{cpu}" + temps.shift + end + + temps.each_with_index do |temp, index| + node.default[:hardware][:sensors][chip][:temps]["temp#{temp}"][:label] = "CPU #{cpu} Core #{index}" + end + end + execute "/etc/sensors.d/chef.conf" do action :nothing command "/usr/bin/sensors -s" @@ -416,7 +597,24 @@ unless Dir.glob("/sys/class/hwmon/hwmon*").empty? source "sensors.conf.erb" owner "root" group "root" - mode 0644 + mode "644" notifies :run, "execute[/etc/sensors.d/chef.conf]" end end + +if node[:hardware][:shm_size] + execute "remount-dev-shm" do + action :nothing + command "/bin/mount -o remount /dev/shm" + user "root" + group "root" + end + + mount "/dev/shm" do + action :enable + device "tmpfs" + fstype "tmpfs" + options "rw,nosuid,nodev,size=#{node[:hardware][:shm_size]}" + notifies :run, "execute[remount-dev-shm]" + end +end