taginfo: renice rebuild to reduce power usage

[chef.git] / cookbooks / taginfo / recipes / default.rb

diff --git a/cookbooks/taginfo/recipes/default.rb b/cookbooks/taginfo/recipes/default.rb
index 7972f380fe851b7464f13ebb4d5afecec6bfe108..080d119241af8a96647d670f2971d5900d73053a 100644 (file)
--- a/cookbooks/taginfo/recipes/default.rb
+++ b/cookbooks/taginfo/recipes/default.rb
@@ -21,8 +21,11 @@ require "json"
 
 include_recipe "accounts"
 include_recipe "apache"
-include_recipe "passenger"
 include_recipe "git"
+include_recipe "passenger"
+include_recipe "planet::current"
+include_recipe "prometheus"
+include_recipe "ruby"
 
 package %w[
   libsqlite3-dev
@@ -43,23 +46,13 @@ package %w[
 
 package %w[
   sqlite3
+  sqlite3-pcre
   osmium-tool
   pyosmium
   curl
   pbzip2
 ]
 
-ruby_version = node[:passenger][:ruby_version]
-
-package "ruby#{ruby_version}"
-
-gem_package "bundler#{ruby_version}" do
-  package_name "bundler"
-  version "~> 1.16.2"
-  gem_binary "gem#{ruby_version}"
-  options "--format-executable"
-end
-
 apache_module "cache"
 apache_module "cache_disk"
 apache_module "headers"
@@ -77,6 +70,28 @@ template "/etc/sudoers.d/taginfo" do
   mode "440"
 end
 
+systemd_service "taginfo-update@" do
+  description "Taginfo update for %i"
+  wants "planet-update.service"
+  after "planet-update.service"
+  exec_start "/srv/%i/bin/update"
+  user "taginfo"
+  sandbox :enable_network => true
+  restrict_address_families "AF_UNIX"
+  read_write_paths [
+    "/srv/%i/data",
+    "/srv/%i/download",
+    "/srv/%i/sources",
+    "/var/log/taginfo/%i"
+  ]
+  nice 10
+end
+
+systemd_timer "taginfo-update@" do
+  description "Taginfo update for %i"
+  on_calendar "01:37"
+end
+
 node[:taginfo][:sites].each do |site|
   site_name = site[:name]
   site_aliases = Array(site[:aliases])
@@ -98,6 +113,33 @@ node[:taginfo][:sites].each do |site|
     mode "755"
   end
 
+  git "#{directory}/taginfo-tools" do
+    action :sync
+    repository "https://github.com/taginfo/taginfo-tools.git"
+    revision "osmorg-taginfo-live"
+    depth 1
+    enable_submodules true
+    user "taginfo"
+    group "taginfo"
+  end
+
+  directory "#{directory}/build" do
+    owner "taginfo"
+    group "taginfo"
+    mode "755"
+  end
+
+  execute "compile_taginfo_tools" do
+    action :nothing
+    user "taginfo"
+    group "taginfo"
+    cwd "#{directory}/build"
+    command "cmake #{directory}/taginfo-tools -DCMAKE_BUILD_TYPE=Release && make"
+    subscribes :run, "apt_package[libprotozero-dev]"
+    subscribes :run, "apt_package[libosmium2-dev]"
+    subscribes :run, "git[#{directory}/taginfo-tools]"
+  end
+
   git "#{directory}/taginfo" do
     action :sync
     repository "https://github.com/taginfo/taginfo.git"
@@ -119,10 +161,11 @@ node[:taginfo][:sites].each do |site|
     settings["logging"]["directory"] = "/var/log/taginfo/#{site_name}"
     settings["opensearch"]["shortname"] = "Taginfo"
     settings["opensearch"]["contact"] = "webmaster@openstreetmap.org"
+    settings["paths"]["bin_dir"] = "#{directory}/build/src"
     settings["sources"]["download"] = ""
-    settings["sources"]["create"] = "db languages projects wiki"
-    settings["sources"]["db"]["planetfile"] = "/var/lib/planet/planet.pbf"
-    settings["sources"]["db"]["bindir"] = "#{directory}/taginfo/tagstats"
+    settings["sources"]["create"] = "db languages projects wiki wikidata chronology"
+    settings["sources"]["db"]["planetfile"] = "/var/lib/planet/planet.osh.pbf"
+    settings["sources"]["chronology"]["osm_history_file"] = "/var/lib/planet/planet.osh.pbf"
     settings["tagstats"]["geodistribution"] = "DenseMmapArray"
 
     JSON.pretty_generate(settings)
@@ -136,25 +179,10 @@ node[:taginfo][:sites].each do |site|
     notifies :restart, "service[apache2]"
   end
 
-  execute "#{directory}/taginfo/tagstats/Makefile" do
+  bundle_install "#{directory}/taginfo" do
     action :nothing
-    command "make"
-    cwd "#{directory}/taginfo/tagstats"
-    user "taginfo"
-    group "taginfo"
-    subscribes :run, "apt_package[libprotozero-dev]"
-    subscribes :run, "apt_package[libosmium2-dev]"
-    subscribes :run, "git[#{directory}/taginfo]"
-    notifies :restart, "service[apache2]"
-  end
-
-  execute "#{directory}/taginfo/Gemfile" do
-    action :nothing
-    command "bundle#{ruby_version} install"
-    cwd "#{directory}/taginfo"
     user "root"
     group "root"
-    subscribes :run, "gem_package[bundler#{ruby_version}]"
     subscribes :run, "git[#{directory}/taginfo]"
     notifies :restart, "passenger_application[#{directory}/taginfo/web/public]"
   end
@@ -189,12 +217,15 @@ node[:taginfo][:sites].each do |site|
     directory "#{directory}/taginfo/web/public"
     variables :aliases => site_aliases
   end
-end
 
-template "/usr/local/bin/taginfo-update" do
-  source "taginfo-update.erb"
-  owner "root"
-  group "root"
-  mode "755"
-  variables :sites => node[:taginfo][:sites]
+  service "taginfo-update@#{site_name}.timer" do
+    action [:enable, :start]
+  end
+
+  prometheus_collector "taginfo-#{site_name}" do
+    interval "15m"
+    user "taginfo"
+    path "#{directory}/taginfo/sources/metrics.rb"
+    options "#{directory}/data"
+  end
 end