From ac030de66ba404119c491c58eede68a4a3325f2d Mon Sep 17 00:00:00 2001
From: Tom Hughes <tom@compton.nu>
Date: Sat, 3 Jul 2021 12:27:51 +0100
Subject: [PATCH] Replace tile log analyser with new one for the new CDN

Fixes #401
---
 cookbooks/tilelog/attributes/default.rb       |  2 -
 cookbooks/tilelog/metadata.rb                 |  3 +-
 cookbooks/tilelog/recipes/default.rb          | 76 ++++---------------
 .../tilelog/templates/default/tilelog.erb     | 29 +++----
 4 files changed, 24 insertions(+), 86 deletions(-)

diff --git a/cookbooks/tilelog/attributes/default.rb b/cookbooks/tilelog/attributes/default.rb
index 3846951a5..9af062fb5 100644
--- a/cookbooks/tilelog/attributes/default.rb
+++ b/cookbooks/tilelog/attributes/default.rb
@@ -1,3 +1 @@
-default[:tilelog][:source_directory] = "/opt/tilelog"
-default[:tilelog][:input_directory] = "/store/logs/tile.openstreetmap.org"
 default[:tilelog][:output_directory] = "/store/planet/tile_logs"
diff --git a/cookbooks/tilelog/metadata.rb b/cookbooks/tilelog/metadata.rb
index 7abc77ea2..b670d0a8b 100644
--- a/cookbooks/tilelog/metadata.rb
+++ b/cookbooks/tilelog/metadata.rb
@@ -6,5 +6,4 @@ description      "Installs and configures tile log analysis"
 
 version          "1.0.0"
 supports         "ubuntu"
-depends          "git"
-depends          "tools"
+depends          "python"
diff --git a/cookbooks/tilelog/recipes/default.rb b/cookbooks/tilelog/recipes/default.rb
index 8b4f91690..7e356e50a 100644
--- a/cookbooks/tilelog/recipes/default.rb
+++ b/cookbooks/tilelog/recipes/default.rb
@@ -17,84 +17,36 @@
 # limitations under the License.
 #
 
-include_recipe "git"
-include_recipe "tools"
+include_recipe "python"
 
-package %w[
-  gcc
-  g++
-  make
-  autoconf
-  automake
-  libboost-filesystem-dev
-  libboost-program-options-dev
-  libboost-system-dev
-]
+passwords = data_bag_item("tilelog", "passwords")
 
-tilelog_source_directory = node[:tilelog][:source_directory]
-tilelog_input_directory = node[:tilelog][:input_directory]
 tilelog_output_directory = node[:tilelog][:output_directory]
 
-# resources for building the tile analysis binary
-git tilelog_source_directory do
-  action :sync
-  repository "https://github.com/zerebubuth/openstreetmap-tile-analyze.git"
-  revision "live"
-  user "root"
-  group "root"
-  notifies :run, "execute[tilelog-autogen]", :immediately
-end
-
-execute "tilelog-autogen" do
-  action :nothing
-  command "autoreconf -i"
-  cwd tilelog_source_directory
-  user "root"
-  group "root"
-  notifies :run, "execute[tilelog-configure]", :immediately
-end
-
-execute "tilelog-configure" do
-  action :nothing
-  command "./configure --with-boost-libdir=/usr/lib/x86_64-linux-gnu"
-  cwd tilelog_source_directory
-  user "root"
-  group "root"
-  notifies :run, "execute[tilelog-build]", :immediately
+python_package "tilelog" do
+  python_version "3"
 end
 
-execute "tilelog-build" do
-  action :nothing
-  command "make"
-  cwd tilelog_source_directory
-  user "root"
-  group "root"
+directory tilelog_output_directory do
+  user "www-data"
+  group "www-data"
+  mode "755"
+  recursive true
 end
 
-# resources for running the tile analysis
-template "/usr/local/bin/tilelog" do
+template "/usr/local/bin/generate-tilelog" do
   source "tilelog.erb"
   owner "root"
   group "root"
   mode "755"
-  variables :analyze_bin => "#{tilelog_source_directory}/openstreetmap-tile-analyze",
-            :input_dir => tilelog_input_directory,
-            :output_dir => tilelog_output_directory
+  variables :output_dir => tilelog_output_directory,
+            :aws_key => passwords["aws_key"]
 end
 
 cron_d "tilelog" do
   minute "17"
   hour "22"
   user "www-data"
-  command "/usr/local/bin/tilelog"
-  mailto "zerebubuth@gmail.com"
-end
-
-# resources related to the output of the analysis and where it
-# can be publicly downloaded.
-directory tilelog_output_directory do
-  user "www-data"
-  group "www-data"
-  mode "755"
-  recursive true
+  command "/usr/local/bin/generate-tilelog"
+  mailto "admins@openstreetmap.org"
 end
diff --git a/cookbooks/tilelog/templates/default/tilelog.erb b/cookbooks/tilelog/templates/default/tilelog.erb
index 598c1e127..f53c69dc2 100644
--- a/cookbooks/tilelog/templates/default/tilelog.erb
+++ b/cookbooks/tilelog/templates/default/tilelog.erb
@@ -1,28 +1,17 @@
 #!/bin/sh
 
-ANALYZE=<%= @analyze_bin %>
-LOGDIR=<%= @input_dir %>
-OUTDIR=<%= @output_dir %>
-if [ -z "$DATE" ]; then
-		DATE=`date -d "1 day ago" "+%Y-%m-%d"`
+if [ -z "$DATE" ]
+then
+  DATE=$(date -d "1 day ago" "+%Y-%m-%d")
 fi
-TMPDIR=`mktemp -d -t tmp.XXXXXXXXX`
-ORIGDIR=`pwd`
 
-clean_up() {
-		cd $ORIGDIR
-		rm -rf $TMPDIR
-}
+OUTDIR="<%= @output_dir %>"
+TMPDIR=$(mktemp -d -t tilelog.XXXXXXXXX)
 
-trap clean_up 0 HUP INT TERM
 cd $TMPDIR
 
-mkdir db
-nice -n 19 $ANALYZE $LOGDIR/*-${DATE}.xz > analyze.log
-if [ -f tiles.txt ]; then
-		nice -n 19 xz -9e -z tiles.txt
-		mv tiles.txt.xz $OUTDIR/tiles-${DATE}.txt.xz
+export AWS_ACCESS_KEY_ID="AKIASQUXHPE7JFCFMOUP"
+export AWS_SECRET_ACCESS_KEY="<%= @aws_key %>"
+export AWS_REGION="eu-west-1"
 
-else
-		cat analyze.log
-fi
+nice -n 19 /usr/local/bin/tilelog --date ${DATE} && mv tiles-${DATE}.txt.xz "${OUTDIR}"
-- 
2.39.5