X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/535456508231fd5fd5e4c3d6671cb7a018b39771..5188a8f5887ab9ec2f08a7a3acf53aad22f38c77:/cookbooks/hardware/recipes/default.rb diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 1a9549d7c..69afc73ff 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -1,8 +1,8 @@ # -# Cookbook Name:: hardware +# Cookbook:: hardware # Recipe:: default # -# Copyright 2012, OpenStreetMap Foundation +# Copyright:: 2012, OpenStreetMap Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,18 +17,27 @@ # limitations under the License. # +include_recipe "apt" +include_recipe "git" +include_recipe "prometheus" +include_recipe "sysfs" include_recipe "tools" -include_recipe "munin" ohai_plugin "hardware" do template "ohai.rb.erb" end -case node[:cpu][:"0"][:vendor_id] -when "GenuineIntel" - package "intel-microcode" -when "AuthenticAMD" - package "amd64-microcode" +if platform?("debian") + package "firmware-linux" +end + +if node[:cpu] && node[:cpu][:"0"] && node[:cpu][:"0"][:vendor_id] + case node[:cpu][:"0"][:vendor_id] + when "GenuineIntel" + package "intel-microcode" + when "AuthenticAMD" + package "amd64-microcode" + end end if node[:dmi] && node[:dmi][:system] @@ -47,79 +56,103 @@ end units = [] -if node[:roles].include?("bytemark") || node[:roles].include?("exonetric") +if node[:roles].include?("bytemark") || node[:roles].include?("exonetric") || node[:roles].include?("prgmr") units << "0" end case manufacturer -when "HP" +when "HP", "HPE" + include_recipe "apt::management-component-pack" + package "hponcfg" + execute "update-ilo" do + action :nothing + command "/usr/sbin/hponcfg -f /etc/ilo-defaults.xml" + not_if { kitchen? } + end + + template "/etc/ilo-defaults.xml" do + source "ilo-defaults.xml.erb" + owner "root" + group "root" + mode "644" + notifies :run, "execute[update-ilo]" + end + package "hp-health" do action :install notifies :restart, "service[hp-health]" + only_if { platform?("ubuntu") && node[:lsb][:release].to_f < 22.04 } end service "hp-health" do action [:enable, :start] supports :status => true, :restart => true + only_if { platform?("ubuntu") && node[:lsb][:release].to_f < 22.04 } end - units << "1" + if product.end_with?("Gen8", "Gen9") + package "hp-ams" do + action :install + notifies :restart, "service[hp-ams]" + end + + service "hp-ams" do + action [:enable, :start] + supports :status => true, :restart => true + end + elsif product.end_with?("Gen10") + package "amsd" do + action :install + notifies :restart, "service[amsd]" + end + + service "amsd" do + action [:enable, :start] + supports :status => true, :restart => true + end + end + + units << if product.end_with?("Gen10") + "0" + else + "1" + end when "TYAN" units << "0" when "TYAN Computer Corporation" units << "0" when "Supermicro" - case product - when "H8DGU", "X9SCD", "X7DBU", "X7DW3", "X9DR7/E-(J)LN4F", "X9DR3-F", "X9DRW", "SYS-1028U-TN10RT+", "SYS-2028U-TN24R4T+", "Super Server" - units << "1" - else - units << "0" - end + units << "1" when "IBM" units << "0" -end +when "VMware, Inc." + package "open-vm-tools" -# Remove legacy HP G4 support which breaks modern hp-health 10.4 -if manufacturer == "HP" - %w[/opt/hp/hp-health/bin/hpasmd /usr/lib/libhpasmintrfc.so.3.0 /usr/lib/libhpasmintrfc.so.3 /usr/lib/libhpasmintrfc.so].each do |filename| - file filename do - action :delete - manage_symlink_source false - end + # Remove timeSync plugin completely + # https://github.com/vmware/open-vm-tools/issues/302 + file "/usr/lib/open-vm-tools/plugins/vmsvc/libtimeSync.so" do + action :delete + notifies :restart, "service[open-vm-tools]" end - directory "/opt/hp/hp-legacy" do - action :delete - recursive true + # Attempt to tell Host we are not interested in timeSync + execute "vmware-toolbox-cmd-timesync-disable" do + command "/usr/bin/vmware-toolbox-cmd timesync disable" + ignore_failure true + end + + service "open-vm-tools" do + action [:enable, :start] + supports :status => true, :restart => true end end units.sort.uniq.each do |unit| - if node[:lsb][:release].to_f >= 16.04 - service "serial-getty@ttyS#{unit}" do - action [:enable, :start] - end - else - file "/etc/init/ttySttyS#{unit}.conf" do - action :delete - end - - template "/etc/init/ttyS#{unit}.conf" do - source "tty.conf.erb" - owner "root" - group "root" - mode 0o644 - variables :unit => unit - end - - service "ttyS#{unit}" do - provider Chef::Provider::Service::Upstart - action [:enable, :start] - supports :status => true, :restart => true, :reload => false - subscribes :restart, "template[/etc/init/ttyS#{unit}.conf]" - end + service "serial-getty@ttyS#{unit}" do + action [:enable, :start] + not_if { kitchen? } end end @@ -133,6 +166,7 @@ if node[:hardware][:grub][:kernel] package "linux-image-#{kernel_version}-generic" package "linux-image-extra-#{kernel_version}-generic" package "linux-headers-#{kernel_version}-generic" + package "linux-tools-#{kernel_version}-generic" boot_device = IO.popen(["df", "/boot"]).readlines.last.split.first boot_uuid = IO.popen(["blkid", "-o", "value", "-s", "UUID", boot_device]).readlines.first.chomp @@ -145,18 +179,21 @@ if File.exist?("/etc/default/grub") execute "update-grub" do action :nothing command "/usr/sbin/update-grub" + not_if { kitchen? } end template "/etc/default/grub" do source "grub.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :units => units, :entry => grub_entry notifies :run, "execute[update-grub]" end end +package "initramfs-tools" + execute "update-initramfs" do action :nothing command "update-initramfs -u -k all" @@ -168,100 +205,134 @@ template "/etc/initramfs-tools/conf.d/mdadm" do source "initramfs-mdadm.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :run, "execute[update-initramfs]" end -package "haveged" -service "haveged" do - action [:enable, :start] +# haveged is only required on older kernels +# /dev/random is not blocking anymore in 5.15+ +if Chef::Util.compare_versions(node[:kernel][:release], [5, 15]).negative? + package "haveged" + service "haveged" do + action [:enable, :start] + end +else + service "haveged" do + action [:stop, :disable] + end + package "haveged" do + action :remove + end end -package "ipmitool" if node[:kernel][:modules].include?("ipmi_si") +watchdog_module = %w[hpwdt sp5100_tco].find do |module_name| + node[:hardware][:pci]&.any? { |_, pci| pci[:modules]&.any?(module_name) } +end -package "irqbalance" +if node[:kernel][:modules].include?("ipmi_si") + package "ipmitool" + package "freeipmi-tools" -template "/etc/default/irqbalance" do - source "irqbalance.erb" - owner "root" - group "root" - mode 0o644 + template "/etc/prometheus/ipmi_local.yml" do + source "ipmi_local.yml.erb" + owner "root" + group "root" + mode "644" + end + + prometheus_exporter "ipmi" do + port 9290 + user "root" + private_devices false + protect_clock false + system_call_filter ["@system-service", "@raw-io"] + options "--config.file=/etc/prometheus/ipmi_local.yml" + subscribes :restart, "template[/etc/prometheus/ipmi_local.yml]" + end + + watchdog_module ||= "ipmi_watchdog" end +package "irqbalance" + service "irqbalance" do action [:start, :enable] supports :status => false, :restart => true, :reload => false - subscribes :restart, "template[/etc/default/irqbalance]" end -# Link Layer Discovery Protocol Daemon package "lldpd" + service "lldpd" do action [:start, :enable] supports :status => true, :restart => true, :reload => true end -if node[:hardware][:mcelog][:enabled] - package "mcelog" +ohai_plugin "lldp" do + template "lldp.rb.erb" +end - %w[bus cache dimm iomca page socket-memory unknown].each do |trigger| - template "/etc/mcelog/#{trigger}-error-trigger.local" do - source "mcelog-trigger.erb" - owner "root" - group "root" - mode 0o755 - end - end +package %w[ + rasdaemon + ruby-sqlite3 +] - service "mcelog" do - action [:start, :enable] - supports :status => true, :restart => true, :reload => false - end +service "rasdaemon" do + action [:enable, :start] +end + +prometheus_exporter "rasdaemon" do + port 9797 + user "root" end tools_packages = [] status_packages = {} -node[:kernel][:modules].each_key do |modname| - case modname - when "cciss" - tools_packages << "ssacli" - status_packages["cciss-vol-status"] ||= [] - when "hpsa" - tools_packages << "ssacli" - status_packages["cciss-vol-status"] ||= [] - when "mptsas" - tools_packages << "lsiutil" - status_packages["mpt-status"] ||= [] - when "mpt2sas", "mpt3sas" - tools_packages << "sas2ircu" - status_packages["sas2ircu-status"] ||= [] - when "megaraid_mm" - tools_packages << "megactl" - status_packages["megaraid-status"] ||= [] - when "megaraid_sas" - tools_packages << "megacli" - status_packages["megaclisas-status"] ||= [] - when "aacraid" - tools_packages << "arcconf" - status_packages["aacraid-status"] ||= [] - when "arcmsr" - tools_packages << "areca" - end -end - -node[:block_device].each do |name, attributes| - next unless attributes[:vendor] == "HP" && attributes[:model] == "LOGICAL VOLUME" - - if name =~ /^cciss!(c[0-9]+)d[0-9]+$/ - status_packages["cciss-vol-status"] |= ["cciss/#{Regexp.last_match[1]}d0"] - else - Dir.glob("/sys/block/#{name}/device/scsi_generic/*").each do |sg| - status_packages["cciss-vol-status"] |= [File.basename(sg)] +if node[:virtualization][:role] != "guest" || + (node[:virtualization][:system] != "lxc" && + node[:virtualization][:system] != "lxd" && + node[:virtualization][:system] != "openvz") + + node[:kernel][:modules].each_key do |modname| + case modname + when "cciss" + tools_packages << "ssacli" + status_packages["cciss-vol-status"] ||= [] + when "hpsa" + tools_packages << "ssacli" + status_packages["cciss-vol-status"] ||= [] + when "mptsas" + tools_packages << "lsiutil" + status_packages["mpt-status"] ||= [] + when "mpt2sas", "mpt3sas" + tools_packages << "sas2ircu" + status_packages["sas2ircu-status"] ||= [] + when "megaraid_sas" + tools_packages << "megacli" + status_packages["megaclisas-status"] ||= [] + when "aacraid" + tools_packages << "arcconf" + status_packages["aacraid-status"] ||= [] + when "arcmsr" + tools_packages << "areca" + end + end + + node[:block_device].each do |name, attributes| + next unless attributes[:vendor] == "HP" && attributes[:model] == "LOGICAL VOLUME" + + if name =~ /^cciss!(c[0-9]+)d[0-9]+$/ + status_packages["cciss-vol-status"] |= ["cciss/#{Regexp.last_match[1]}d0"] + else + Dir.glob("/sys/block/#{name}/device/scsi_generic/*").each do |sg| + status_packages["cciss-vol-status"] |= [File.basename(sg)] + end end end end +include_recipe "apt::hwraid" unless status_packages.empty? + %w[ssacli lsiutil sas2ircu megactl megacli arcconf].each do |tools_package| if tools_packages.include?(tools_package) package tools_package @@ -278,8 +349,10 @@ if tools_packages.include?("areca") git "/opt/areca" do action :sync repository "https://git.openstreetmap.org/private/areca.git" + depth 1 user "root" group "root" + not_if { kitchen? } end else directory "/opt/areca" do @@ -288,34 +361,32 @@ else end end -["cciss-vol-status", "mpt-status", "sas2ircu-status", "megaraid-status", "megaclisas-status", "aacraid-status"].each do |status_package| +%w[cciss-vol-status mpt-status sas2ircu-status megaclisas-status aacraid-status].each do |status_package| if status_packages.include?(status_package) package status_package - template "/etc/default/#{status_package}d" do - source "raid.default.erb" - owner "root" - group "root" - mode 0o644 - variables :devices => status_packages[status_package] + service "#{status_package}d" do + action [:stop, :disable] end - service "#{status_package}d" do - action [:start, :enable] - supports :status => false, :restart => true, :reload => false - subscribes :restart, "template[/etc/default/#{status_package}d]" + file "/etc/default/#{status_package}d" do + action :delete end else package status_package do action :purge end - - file "/etc/default/#{status_package}d" do - action :delete - end end end +systemd_service "cciss-vol-statusd" do + action :delete +end + +template "/usr/local/bin/cciss-vol-statusd" do + action :delete +end + disks = if node[:hardware][:disk] node[:hardware][:disk][:disks] else @@ -330,31 +401,42 @@ nvmes = if node[:hardware][:pci] [] end +unless nvmes.empty? + package "nvme-cli" +end + intel_nvmes = nvmes.select { |pci| pci[:vendor_name] == "Intel Corporation" } if !intel_ssds.empty? || !intel_nvmes.empty? package "unzip" - remote_file "#{Chef::Config[:file_cache_path]}/Intel_SSD_Data_Center_Tool_3.0.13_Linux.zip" do - source "https://downloadmirror.intel.com/27863/eng/Intel_SSD_Data_Center_Tool_3.0.13_Linux.zip" + sst_tool_version = "2-0" + sst_package_version = "2.0.300-0" + + remote_file "#{Chef::Config[:file_cache_path]}/sst-cli-linux-deb--#{sst_tool_version}.zip" do + source "https://sdmsdfwdriver.blob.core.windows.net/files/kba-gcc/drivers-downloads/ka-00085/sst--#{sst_tool_version}/sst-cli-linux-deb--#{sst_tool_version}.zip" end - execute "#{Chef::Config[:file_cache_path]}/Intel_SSD_Data_Center_Tool_3.0.13_Linux.zip" do - command "unzip Intel_SSD_Data_Center_Tool_3.0.13_Linux.zip isdct_3.0.13.400-17_amd64.deb" + execute "#{Chef::Config[:file_cache_path]}/sst-cli-linux-deb--#{sst_tool_version}.zip" do + command "unzip sst-cli-linux-deb--#{sst_tool_version}.zip sst_#{sst_package_version}_amd64.deb" cwd Chef::Config[:file_cache_path] user "root" group "root" - not_if { File.exist?("#{Chef::Config[:file_cache_path]}/isdct_3.0.13.400-17_amd64.deb") } + not_if { ::File.exist?("#{Chef::Config[:file_cache_path]}/sst_#{sst_package_version}_amd64.deb") } end - dpkg_package "isdct" do - version "3.0.13.400-17" - source "#{Chef::Config[:file_cache_path]}/isdct_3.0.13.400-17_amd64.deb" + dpkg_package "sst" do + version "#{sst_package_version}" + source "#{Chef::Config[:file_cache_path]}/sst_#{sst_package_version}_amd64.deb" + end + + dpkg_package "intelmas" do + action :purge end end disks = disks.map do |disk| - next if disk[:state] == "spun_down" + next if disk[:state] == "spun_down" || %w[unconfigured failed].any?(disk[:status]) if disk[:smart_device] controller = node[:hardware][:disk][:controllers][disk[:controller]] @@ -362,54 +444,48 @@ disks = disks.map do |disk| if controller && controller[:device] device = controller[:device].sub("/dev/", "") smart = disk[:smart_device] - - if device.start_with?("cciss/") && smart =~ /^cciss,(\d+)$/ - array = node[:hardware][:disk][:arrays][disk[:arrays].first] - munin = "cciss-3#{array[:wwn]}-#{Regexp.last_match(1)}" - elsif smart =~ /^.*,(\d+)$/ - munin = "#{device}-#{Regexp.last_match(1)}" - elsif smart =~ %r{^.*,(\d+)/(\d+)$} - munin = "#{device}-#{Regexp.last_match(1)}:#{Regexp.last_match(2)}" - end + elsif disk[:device] + device = disk[:device].sub("/dev/", "") + smart = disk[:smart_device] end + elsif disk[:device] =~ %r{^/dev/(nvme\d+)n\d+$} + device = Regexp.last_match(1) elsif disk[:device] device = disk[:device].sub("/dev/", "") - munin = device end next if device.nil? Hash[ :device => device, - :smart => smart, - :munin => munin, - :hddtemp => munin.tr("-:", "_") + :smart => smart ] end -smartd_service = if node[:lsb][:release].to_f >= 16.04 - "smartd" - else - "smartmontools" - end - -disks = disks.compact +disks = disks.compact.uniq if disks.count.positive? package "smartmontools" + template "/etc/cron.daily/update-smart-drivedb" do + source "update-smart-drivedb.erb" + owner "root" + group "root" + mode "755" + end + template "/usr/local/bin/smartd-mailer" do source "smartd-mailer.erb" owner "root" group "root" - mode 0o755 + mode "755" end template "/etc/smartd.conf" do source "smartd.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :disks => disks end @@ -417,52 +493,34 @@ if disks.count.positive? source "smartmontools.erb" owner "root" group "root" - mode 0o644 + mode "644" end - service smartd_service do + service "smartmontools" do action [:enable, :start] subscribes :reload, "template[/etc/smartd.conf]" subscribes :restart, "template[/etc/default/smartmontools]" end - # Don't try and do munin monitoring of disks behind - # an Areca controller as they only allow one thing to - # talk to the controller at a time and smartd will - # throw errors if it clashes with munin - disks = disks.reject { |disk| disk[:smart]&.start_with?("areca,") } - - disks.each do |disk| - munin_plugin "smart_#{disk[:munin]}" do - target "smart_" - conf "munin.smart.erb" - conf_variables :disk => disk - end - end -else - service smartd_service do - action [:stop, :disable] + template "/etc/prometheus/collectors/smart.devices" do + source "smart.devices.erb" + owner "root" + group "root" + mode "644" + variables :disks => disks end -end -if disks.count.positive? - munin_plugin "hddtemp_smartctl" do - conf "munin.hddtemp.erb" - conf_variables :disks => disks + prometheus_collector "smart" do + interval "15m" + user "root" + capability_bounding_set %w[CAP_DAC_OVERRIDE CAP_SYS_ADMIN CAP_SYS_RAWIO] + private_devices false + private_users false + protect_clock false end else - munin_plugin "hddtemp_smartctl" do - action :delete - conf "munin.hddtemp.erb" - end -end - -plugins = Dir.glob("/etc/munin/plugins/smart_*").map { |p| File.basename(p) } - - disks.map { |d| "smart_#{d[:munin]}" } - -plugins.each do |plugin| - munin_plugin plugin do - action :delete + service "smartd" do + action [:stop, :disable] end end @@ -476,41 +534,57 @@ if File.exist?("/etc/mdadm/mdadm.conf") file "/etc/mdadm/mdadm.conf" do owner "root" group "root" - mode 0o644 + mode "644" content mdadm_conf end - service "mdadm" do + service "mdmonitor" do action :nothing subscribes :restart, "file[/etc/mdadm/mdadm.conf]" end end -template "/etc/modules" do - source "modules.erb" - owner "root" - group "root" - mode 0o644 +file "/etc/modules" do + action :delete end -service "kmod" do - action :nothing - subscribes :start, "template[/etc/modules]" +node[:hardware][:modules].each do |module_name| + kernel_module module_name do + action :install + not_if { kitchen? } + end end -if node[:hardware][:watchdog] - package "watchdog" +node[:hardware][:blacklisted_modules].each do |module_name| + kernel_module module_name do + action :blacklist + end +end + +if watchdog_module + kernel_module watchdog_module do + action :install + end + + execute "systemctl-reload" do + action :nothing + command "systemctl daemon-reload" + user "root" + group "root" + end - template "/etc/default/watchdog" do - source "watchdog.erb" + directory "/etc/systemd/system.conf.d" do owner "root" group "root" - mode 0o644 - variables :module => node[:hardware][:watchdog] + mode "755" end - service "watchdog" do - action [:enable, :start] + template "/etc/systemd/system.conf.d/watchdog.conf" do + source "watchdog.conf.erb" + owner "root" + group "root" + mode "644" + notifies :run, "execute[systemctl-reload]" end end @@ -552,7 +626,35 @@ unless Dir.glob("/sys/class/hwmon/hwmon*").empty? source "sensors.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :run, "execute[/etc/sensors.d/chef.conf]" end end + +if node[:hardware][:shm_size] + execute "remount-dev-shm" do + action :nothing + command "/bin/mount -o remount /dev/shm" + user "root" + group "root" + end + + mount "/dev/shm" do + action :enable + device "tmpfs" + fstype "tmpfs" + options "rw,nosuid,nodev,size=#{node[:hardware][:shm_size]}" + notifies :run, "execute[remount-dev-shm]" + end +end + +prometheus_collector "ohai" do + interval "15m" + user "root" + proc_subset "all" + capability_bounding_set %w[CAP_DAC_OVERRIDE CAP_SYS_ADMIN CAP_SYS_RAWIO] + private_devices false + private_users false + protect_clock false + protect_kernel_modules false +end