Add a planet::dump recipe and apply it to smaug

diff --git a/cookbooks/planet/recipes/dump.rb b/cookbooks/planet/recipes/dump.rb
new file mode 100644 (file)
index 0000000..197ed10
--- /dev/null
+++ b/cookbooks/planet/recipes/dump.rb
@@ -0,0 +1,64 @@
+#
+# Cookbook Name:: planet
+# Recipe:: dump
+#
+# Copyright 2013, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "git"
+
+db_passwords = data_bag_item("db", "passwords")
+
+package "gcc"
+package "make"
+package "libpqxx3-dev"
+
+directory "/opt/planetdump" do
+  owner "root"
+  group "root"
+  mode 0755
+end
+
+git "/opt/planetdump" do
+  action :sync
+  repository "git://git.openstreetmap.org/planetdump.git"
+  revision "live"
+  user "root"
+  group "root"
+end
+
+execute "/opt/planetdump/Makefile" do
+  action :nothing
+  command "make planet06_pg"
+  cwd "/opt/planetdump"
+  user "root"
+  group "root"
+  subscribes :run, "git[/opt/planetdump]"
+end
+
+template "/usr/local/bin/planetdump" do
+  source "planetdump.erb"
+  owner "root"
+  group "root"
+  mode 0755
+  variables :password => db_passwords["planetdump"]
+end
+
+template "/etc/cron.d/planetdump" do
+  source "planetdump.cron.erb"
+  owner "root"
+  group "root"
+  mode 0644
+end

diff --git a/cookbooks/planet/templates/default/planetdump.cron.erb b/cookbooks/planet/templates/default/planetdump.cron.erb
new file mode 100644 (file)
index 0000000..b7403d2
--- /dev/null
+++ b/cookbooks/planet/templates/default/planetdump.cron.erb
@@ -0,0 +1,2 @@
+MAILTO=grant-smaug@firefishy.com
+10 0 * * 3 www-data /usr/local/bin/planetdump

diff --git a/cookbooks/planet/templates/default/planetdump.erb b/cookbooks/planet/templates/default/planetdump.erb
new file mode 100644 (file)
index 0000000..fbadac8
--- /dev/null
+++ b/cookbooks/planet/templates/default/planetdump.erb
@@ -0,0 +1,71 @@
+#!/bin/bash
+ulimit -m 4194304 -v 4194304
+set -e
+
+if [ -f /tmp/planetdump.lock ]; then
+       if [ "$(ps -p `cat /tmp/planetdump.lock` | wc -l)" -gt 1 ]; then
+               # process is still running
+               echo Error: Another planetdump is running
+               exit 1
+       else
+               # process not running, but lock file not deleted?
+               rm /tmp/planetdump.lock
+       fi
+fi
+
+#Create Lock
+echo $$ > /tmp/planetdump.lock
+
+
+cur_date=`date +%y%m%d`
+cur_year=`date +%Y`
+cur_planet=planet-${cur_date}.osm
+cur_changeset=changesets-${cur_date}.osm
+planet_dir=/store/planet/
+
+export CONNECTION_PARAMS='dbname=openstreetmap host=localhost user=planetdump password=<%= @password %>'
+export PATH='/usr/local/bin:/usr/bin:/bin:/usr/bin/X11'
+
+
+if [ \! -d ${planet_dir}/planet/ ]
+       then
+               echo ${planet_dir}planet/ does not exist
+               exit 1
+       fi
+if [ \! -d ${planet_dir}/planet/${cur_year}/ ]; then mkdir ${planet_dir}/planet/${cur_year}/; fi
+cd ${planet_dir}/planet/${cur_year}/
+
+/opt/planetdump/planet06_pg | pbzip2 -p6 -7c > .${cur_planet}.bz2.new
+/opt/planetdump/planet06_pg --changesets | pbzip2 -p6 -9c > .${cur_changeset}.bz2.new
+
+planet_size=$(du -sb .${cur_planet}.bz2.new | awk '{ print $1 }')
+changeset_size=$(du -sb .${cur_changeset}.bz2.new | awk '{ print $1 }')
+if ((planet_size<28000000000)); then 
+       echo Planet .${cur_planet}.bz2.new too small
+       exit 1
+fi
+if ((changeset_size<600000000)); then
+       echo Changeset .${cur_changeset}.bz2.new too small
+       exit 1
+fi
+
+mv .${cur_planet}.bz2.new ${cur_planet}.bz2
+mv .${cur_changeset}.bz2.new ${cur_changeset}.bz2
+md5sum ${cur_planet}.bz2 > ${cur_planet}.bz2.md5
+md5sum ${cur_changeset}.bz2 > ${cur_changeset}.bz2.md5
+
+#link planet latest to the new file
+cd ${planet_dir}/planet/
+
+ln -fs ${cur_year}/${cur_planet}.bz2 planet-latest.osm.bz2
+ln -fs ${cur_year}/${cur_changeset}.bz2 changesets-latest.osm.bz2
+
+# mangle md5 files for 'latest' ones
+rm -f planet-latest.osm.bz2.md5
+rm -f changesets-latest.osm.bz2.md5
+
+sed -e "s/${cur_planet}.bz2/planet-latest.osm.bz2/" ${cur_year}/${cur_planet}.bz2.md5 > planet-latest.osm.bz2.md5
+sed -e "s/${cur_changeset}.bz2/changesets-latest.osm.bz2/" ${cur_year}/${cur_changeset}.bz2.md5 > changesets-latest.osm.bz2.md5
+
+rm /tmp/planetdump.lock
+

diff --git a/roles/planetdump.rb b/roles/planetdump.rb
new file mode 100644 (file)
index 0000000..d48e5cb
--- /dev/null
+++ b/roles/planetdump.rb
@@ -0,0 +1,6 @@
+name "planetdump"
+description "Role applied to all planetdump servers"
+
+run_list(
+  "recipe[planet::dump]"
+)

diff --git a/roles/smaug.rb b/roles/smaug.rb
index 122c4d3140a46b43c151d023740055546380e732..9c6d33b05afe3771bf4a387155f06d098e9ac4f7 100644 (file)
--- a/roles/smaug.rb
+++ b/roles/smaug.rb
@@ -73,5 +73,6 @@ default_attributes(
 
 run_list(
   "role[ic]",
-  "role[db-slave]"
+  "role[db-slave]",
+  "role[planetdump]"
 )