From: Tom Hughes <tom@compton.nu>
Date: Wed, 19 Jun 2013 17:55:51 +0000 (+0100)
Subject: Add a planet::dump recipe and apply it to smaug
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/932f7a837e84c5c29fd8429a574dd7aea5314b0a

Add a planet::dump recipe and apply it to smaug
---

diff --git a/cookbooks/planet/recipes/dump.rb b/cookbooks/planet/recipes/dump.rb
new file mode 100644
index 000000000..197ed10b4
--- /dev/null
+++ b/cookbooks/planet/recipes/dump.rb
@@ -0,0 +1,64 @@
+#
+# Cookbook Name:: planet
+# Recipe:: dump
+#
+# Copyright 2013, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "git"
+
+db_passwords = data_bag_item("db", "passwords")
+
+package "gcc"
+package "make"
+package "libpqxx3-dev"
+
+directory "/opt/planetdump" do
+  owner "root"
+  group "root"
+  mode 0755
+end
+
+git "/opt/planetdump" do
+  action :sync
+  repository "git://git.openstreetmap.org/planetdump.git"
+  revision "live"
+  user "root"
+  group "root"
+end
+
+execute "/opt/planetdump/Makefile" do
+  action :nothing
+  command "make planet06_pg"
+  cwd "/opt/planetdump"
+  user "root"
+  group "root"
+  subscribes :run, "git[/opt/planetdump]"
+end
+
+template "/usr/local/bin/planetdump" do
+  source "planetdump.erb"
+  owner "root"
+  group "root"
+  mode 0755
+  variables :password => db_passwords["planetdump"]
+end
+
+template "/etc/cron.d/planetdump" do
+  source "planetdump.cron.erb"
+  owner "root"
+  group "root"
+  mode 0644
+end
diff --git a/cookbooks/planet/templates/default/planetdump.cron.erb b/cookbooks/planet/templates/default/planetdump.cron.erb
new file mode 100644
index 000000000..b7403d22c
--- /dev/null
+++ b/cookbooks/planet/templates/default/planetdump.cron.erb
@@ -0,0 +1,2 @@
+MAILTO=grant-smaug@firefishy.com
+10 0 * * 3 www-data /usr/local/bin/planetdump
diff --git a/cookbooks/planet/templates/default/planetdump.erb b/cookbooks/planet/templates/default/planetdump.erb
new file mode 100644
index 000000000..fbadac8b9
--- /dev/null
+++ b/cookbooks/planet/templates/default/planetdump.erb
@@ -0,0 +1,71 @@
+#!/bin/bash
+ulimit -m 4194304 -v 4194304
+set -e
+
+if [ -f /tmp/planetdump.lock ]; then
+	if [ "$(ps -p `cat /tmp/planetdump.lock` | wc -l)" -gt 1 ]; then
+		# process is still running
+		echo Error: Another planetdump is running
+		exit 1
+	else
+		# process not running, but lock file not deleted?
+		rm /tmp/planetdump.lock
+	fi
+fi
+
+#Create Lock
+echo $$ > /tmp/planetdump.lock
+
+
+cur_date=`date +%y%m%d`
+cur_year=`date +%Y`
+cur_planet=planet-${cur_date}.osm
+cur_changeset=changesets-${cur_date}.osm
+planet_dir=/store/planet/
+
+export CONNECTION_PARAMS='dbname=openstreetmap host=localhost user=planetdump password=<%= @password %>'
+export PATH='/usr/local/bin:/usr/bin:/bin:/usr/bin/X11'
+
+
+if [ \! -d ${planet_dir}/planet/ ]
+	then
+		echo ${planet_dir}planet/ does not exist
+		exit 1
+	fi
+if [ \! -d ${planet_dir}/planet/${cur_year}/ ]; then mkdir ${planet_dir}/planet/${cur_year}/; fi
+cd ${planet_dir}/planet/${cur_year}/
+
+/opt/planetdump/planet06_pg | pbzip2 -p6 -7c > .${cur_planet}.bz2.new
+/opt/planetdump/planet06_pg --changesets | pbzip2 -p6 -9c > .${cur_changeset}.bz2.new
+
+planet_size=$(du -sb .${cur_planet}.bz2.new | awk '{ print $1 }')
+changeset_size=$(du -sb .${cur_changeset}.bz2.new | awk '{ print $1 }')
+if ((planet_size<28000000000)); then 
+	echo Planet .${cur_planet}.bz2.new too small
+	exit 1
+fi
+if ((changeset_size<600000000)); then
+	echo Changeset .${cur_changeset}.bz2.new too small
+	exit 1
+fi
+
+mv .${cur_planet}.bz2.new ${cur_planet}.bz2
+mv .${cur_changeset}.bz2.new ${cur_changeset}.bz2
+md5sum ${cur_planet}.bz2 > ${cur_planet}.bz2.md5
+md5sum ${cur_changeset}.bz2 > ${cur_changeset}.bz2.md5
+
+#link planet latest to the new file
+cd ${planet_dir}/planet/
+
+ln -fs ${cur_year}/${cur_planet}.bz2 planet-latest.osm.bz2
+ln -fs ${cur_year}/${cur_changeset}.bz2 changesets-latest.osm.bz2
+
+# mangle md5 files for 'latest' ones
+rm -f planet-latest.osm.bz2.md5
+rm -f changesets-latest.osm.bz2.md5
+
+sed -e "s/${cur_planet}.bz2/planet-latest.osm.bz2/" ${cur_year}/${cur_planet}.bz2.md5 > planet-latest.osm.bz2.md5
+sed -e "s/${cur_changeset}.bz2/changesets-latest.osm.bz2/" ${cur_year}/${cur_changeset}.bz2.md5 > changesets-latest.osm.bz2.md5
+
+rm /tmp/planetdump.lock
+
diff --git a/roles/planetdump.rb b/roles/planetdump.rb
new file mode 100644
index 000000000..d48e5cbce
--- /dev/null
+++ b/roles/planetdump.rb
@@ -0,0 +1,6 @@
+name "planetdump"
+description "Role applied to all planetdump servers"
+
+run_list(
+  "recipe[planet::dump]"
+)
diff --git a/roles/smaug.rb b/roles/smaug.rb
index 122c4d314..9c6d33b05 100644
--- a/roles/smaug.rb
+++ b/roles/smaug.rb
@@ -73,5 +73,6 @@ default_attributes(
 
 run_list(
   "role[ic]",
-  "role[db-slave]"
+  "role[db-slave]",
+  "role[planetdump]"
 )